1 /* 2 * Copyright 2020 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #include <linux/debugfs.h> 26 #include <linux/io-mapping.h> 27 #include <linux/iosys-map.h> 28 #include <linux/scatterlist.h> 29 30 #include <drm/ttm/ttm_bo.h> 31 #include <drm/ttm/ttm_placement.h> 32 #include <drm/ttm/ttm_resource.h> 33 34 #include <drm/drm_util.h> 35 36 /** 37 * ttm_lru_bulk_move_init - initialize a bulk move structure 38 * @bulk: the structure to init 39 * 40 * For now just memset the structure to zero. 41 */ 42 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk) 43 { 44 memset(bulk, 0, sizeof(*bulk)); 45 } 46 EXPORT_SYMBOL(ttm_lru_bulk_move_init); 47 48 /** 49 * ttm_lru_bulk_move_tail - bulk move range of resources to the LRU tail. 50 * 51 * @bulk: bulk move structure 52 * 53 * Bulk move BOs to the LRU tail, only valid to use when driver makes sure that 54 * resource order never changes. Should be called with &ttm_device.lru_lock held. 55 */ 56 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk) 57 { 58 unsigned i, j; 59 60 for (i = 0; i < TTM_NUM_MEM_TYPES; ++i) { 61 for (j = 0; j < TTM_MAX_BO_PRIORITY; ++j) { 62 struct ttm_lru_bulk_move_pos *pos = &bulk->pos[i][j]; 63 struct ttm_resource_manager *man; 64 65 if (!pos->first) 66 continue; 67 68 lockdep_assert_held(&pos->first->bo->bdev->lru_lock); 69 dma_resv_assert_held(pos->first->bo->base.resv); 70 dma_resv_assert_held(pos->last->bo->base.resv); 71 72 man = ttm_manager_type(pos->first->bo->bdev, i); 73 list_bulk_move_tail(&man->lru[j], &pos->first->lru.link, 74 &pos->last->lru.link); 75 } 76 } 77 } 78 EXPORT_SYMBOL(ttm_lru_bulk_move_tail); 79 80 /* Return the bulk move pos object for this resource */ 81 static struct ttm_lru_bulk_move_pos * 82 ttm_lru_bulk_move_pos(struct ttm_lru_bulk_move *bulk, struct ttm_resource *res) 83 { 84 return &bulk->pos[res->mem_type][res->bo->priority]; 85 } 86 87 /* Return the previous resource on the list (skip over non-resource list items) */ 88 static struct ttm_resource *ttm_lru_prev_res(struct ttm_resource *cur) 89 { 90 struct ttm_lru_item *lru = &cur->lru; 91 92 do { 93 lru = list_prev_entry(lru, link); 94 } while (!ttm_lru_item_is_res(lru)); 95 96 return ttm_lru_item_to_res(lru); 97 } 98 99 /* Return the next resource on the list (skip over non-resource list items) */ 100 static struct ttm_resource *ttm_lru_next_res(struct ttm_resource *cur) 101 { 102 struct ttm_lru_item *lru = &cur->lru; 103 104 do { 105 lru = list_next_entry(lru, link); 106 } while (!ttm_lru_item_is_res(lru)); 107 108 return ttm_lru_item_to_res(lru); 109 } 110 111 /* Move the resource to the tail of the bulk move range */ 112 static void ttm_lru_bulk_move_pos_tail(struct ttm_lru_bulk_move_pos *pos, 113 struct ttm_resource *res) 114 { 115 if (pos->last != res) { 116 if (pos->first == res) 117 pos->first = ttm_lru_next_res(res); 118 list_move(&res->lru.link, &pos->last->lru.link); 119 pos->last = res; 120 } 121 } 122 123 /* Add the resource to a bulk_move cursor */ 124 static void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk, 125 struct ttm_resource *res) 126 { 127 struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res); 128 129 if (!pos->first) { 130 pos->first = res; 131 pos->last = res; 132 } else { 133 WARN_ON(pos->first->bo->base.resv != res->bo->base.resv); 134 ttm_lru_bulk_move_pos_tail(pos, res); 135 } 136 } 137 138 /* Remove the resource from a bulk_move range */ 139 static void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk, 140 struct ttm_resource *res) 141 { 142 struct ttm_lru_bulk_move_pos *pos = ttm_lru_bulk_move_pos(bulk, res); 143 144 if (unlikely(WARN_ON(!pos->first || !pos->last) || 145 (pos->first == res && pos->last == res))) { 146 pos->first = NULL; 147 pos->last = NULL; 148 } else if (pos->first == res) { 149 pos->first = ttm_lru_next_res(res); 150 } else if (pos->last == res) { 151 pos->last = ttm_lru_prev_res(res); 152 } else { 153 list_move(&res->lru.link, &pos->last->lru.link); 154 } 155 } 156 157 /* Add the resource to a bulk move if the BO is configured for it */ 158 void ttm_resource_add_bulk_move(struct ttm_resource *res, 159 struct ttm_buffer_object *bo) 160 { 161 if (bo->bulk_move && !bo->pin_count) 162 ttm_lru_bulk_move_add(bo->bulk_move, res); 163 } 164 165 /* Remove the resource from a bulk move if the BO is configured for it */ 166 void ttm_resource_del_bulk_move(struct ttm_resource *res, 167 struct ttm_buffer_object *bo) 168 { 169 if (bo->bulk_move && !bo->pin_count) 170 ttm_lru_bulk_move_del(bo->bulk_move, res); 171 } 172 173 /* Move a resource to the LRU or bulk tail */ 174 void ttm_resource_move_to_lru_tail(struct ttm_resource *res) 175 { 176 struct ttm_buffer_object *bo = res->bo; 177 struct ttm_device *bdev = bo->bdev; 178 179 lockdep_assert_held(&bo->bdev->lru_lock); 180 181 if (bo->pin_count) { 182 list_move_tail(&res->lru.link, &bdev->pinned); 183 184 } else if (bo->bulk_move) { 185 struct ttm_lru_bulk_move_pos *pos = 186 ttm_lru_bulk_move_pos(bo->bulk_move, res); 187 188 ttm_lru_bulk_move_pos_tail(pos, res); 189 } else { 190 struct ttm_resource_manager *man; 191 192 man = ttm_manager_type(bdev, res->mem_type); 193 list_move_tail(&res->lru.link, &man->lru[bo->priority]); 194 } 195 } 196 197 /** 198 * ttm_resource_init - resource object constructure 199 * @bo: buffer object this resources is allocated for 200 * @place: placement of the resource 201 * @res: the resource object to inistilize 202 * 203 * Initialize a new resource object. Counterpart of ttm_resource_fini(). 204 */ 205 void ttm_resource_init(struct ttm_buffer_object *bo, 206 const struct ttm_place *place, 207 struct ttm_resource *res) 208 { 209 struct ttm_resource_manager *man; 210 211 res->start = 0; 212 res->size = bo->base.size; 213 res->mem_type = place->mem_type; 214 res->placement = place->flags; 215 res->bus.addr = NULL; 216 res->bus.offset = 0; 217 res->bus.is_iomem = false; 218 res->bus.caching = ttm_cached; 219 res->bo = bo; 220 221 man = ttm_manager_type(bo->bdev, place->mem_type); 222 spin_lock(&bo->bdev->lru_lock); 223 if (bo->pin_count) 224 list_add_tail(&res->lru.link, &bo->bdev->pinned); 225 else 226 list_add_tail(&res->lru.link, &man->lru[bo->priority]); 227 man->usage += res->size; 228 spin_unlock(&bo->bdev->lru_lock); 229 } 230 EXPORT_SYMBOL(ttm_resource_init); 231 232 /** 233 * ttm_resource_fini - resource destructor 234 * @man: the resource manager this resource belongs to 235 * @res: the resource to clean up 236 * 237 * Should be used by resource manager backends to clean up the TTM resource 238 * objects before freeing the underlying structure. Makes sure the resource is 239 * removed from the LRU before destruction. 240 * Counterpart of ttm_resource_init(). 241 */ 242 void ttm_resource_fini(struct ttm_resource_manager *man, 243 struct ttm_resource *res) 244 { 245 struct ttm_device *bdev = man->bdev; 246 247 spin_lock(&bdev->lru_lock); 248 list_del_init(&res->lru.link); 249 man->usage -= res->size; 250 spin_unlock(&bdev->lru_lock); 251 } 252 EXPORT_SYMBOL(ttm_resource_fini); 253 254 int ttm_resource_alloc(struct ttm_buffer_object *bo, 255 const struct ttm_place *place, 256 struct ttm_resource **res_ptr) 257 { 258 struct ttm_resource_manager *man = 259 ttm_manager_type(bo->bdev, place->mem_type); 260 int ret; 261 262 ret = man->func->alloc(man, bo, place, res_ptr); 263 if (ret) 264 return ret; 265 266 spin_lock(&bo->bdev->lru_lock); 267 ttm_resource_add_bulk_move(*res_ptr, bo); 268 spin_unlock(&bo->bdev->lru_lock); 269 return 0; 270 } 271 EXPORT_SYMBOL_FOR_TESTS_ONLY(ttm_resource_alloc); 272 273 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res) 274 { 275 struct ttm_resource_manager *man; 276 277 if (!*res) 278 return; 279 280 spin_lock(&bo->bdev->lru_lock); 281 ttm_resource_del_bulk_move(*res, bo); 282 spin_unlock(&bo->bdev->lru_lock); 283 man = ttm_manager_type(bo->bdev, (*res)->mem_type); 284 man->func->free(man, *res); 285 *res = NULL; 286 } 287 EXPORT_SYMBOL(ttm_resource_free); 288 289 /** 290 * ttm_resource_intersects - test for intersection 291 * 292 * @bdev: TTM device structure 293 * @res: The resource to test 294 * @place: The placement to test 295 * @size: How many bytes the new allocation needs. 296 * 297 * Test if @res intersects with @place and @size. Used for testing if evictions 298 * are valueable or not. 299 * 300 * Returns true if the res placement intersects with @place and @size. 301 */ 302 bool ttm_resource_intersects(struct ttm_device *bdev, 303 struct ttm_resource *res, 304 const struct ttm_place *place, 305 size_t size) 306 { 307 struct ttm_resource_manager *man; 308 309 if (!res) 310 return false; 311 312 man = ttm_manager_type(bdev, res->mem_type); 313 if (!place || !man->func->intersects) 314 return true; 315 316 return man->func->intersects(man, res, place, size); 317 } 318 319 /** 320 * ttm_resource_compatible - check if resource is compatible with placement 321 * 322 * @res: the resource to check 323 * @placement: the placement to check against 324 * @evicting: true if the caller is doing evictions 325 * 326 * Returns true if the placement is compatible. 327 */ 328 bool ttm_resource_compatible(struct ttm_resource *res, 329 struct ttm_placement *placement, 330 bool evicting) 331 { 332 struct ttm_buffer_object *bo = res->bo; 333 struct ttm_device *bdev = bo->bdev; 334 unsigned i; 335 336 if (res->placement & TTM_PL_FLAG_TEMPORARY) 337 return false; 338 339 for (i = 0; i < placement->num_placement; i++) { 340 const struct ttm_place *place = &placement->placement[i]; 341 struct ttm_resource_manager *man; 342 343 if (res->mem_type != place->mem_type) 344 continue; 345 346 if (place->flags & (evicting ? TTM_PL_FLAG_DESIRED : 347 TTM_PL_FLAG_FALLBACK)) 348 continue; 349 350 if (place->flags & TTM_PL_FLAG_CONTIGUOUS && 351 !(res->placement & TTM_PL_FLAG_CONTIGUOUS)) 352 continue; 353 354 man = ttm_manager_type(bdev, res->mem_type); 355 if (man->func->compatible && 356 !man->func->compatible(man, res, place, bo->base.size)) 357 continue; 358 359 return true; 360 } 361 return false; 362 } 363 364 void ttm_resource_set_bo(struct ttm_resource *res, 365 struct ttm_buffer_object *bo) 366 { 367 spin_lock(&bo->bdev->lru_lock); 368 res->bo = bo; 369 spin_unlock(&bo->bdev->lru_lock); 370 } 371 372 /** 373 * ttm_resource_manager_init 374 * 375 * @man: memory manager object to init 376 * @bdev: ttm device this manager belongs to 377 * @size: size of managed resources in arbitrary units 378 * 379 * Initialise core parts of a manager object. 380 */ 381 void ttm_resource_manager_init(struct ttm_resource_manager *man, 382 struct ttm_device *bdev, 383 uint64_t size) 384 { 385 unsigned i; 386 387 spin_lock_init(&man->move_lock); 388 man->bdev = bdev; 389 man->size = size; 390 man->usage = 0; 391 392 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) 393 INIT_LIST_HEAD(&man->lru[i]); 394 man->move = NULL; 395 } 396 EXPORT_SYMBOL(ttm_resource_manager_init); 397 398 /* 399 * ttm_resource_manager_evict_all 400 * 401 * @bdev - device to use 402 * @man - manager to use 403 * 404 * Evict all the objects out of a memory manager until it is empty. 405 * Part of memory manager cleanup sequence. 406 */ 407 int ttm_resource_manager_evict_all(struct ttm_device *bdev, 408 struct ttm_resource_manager *man) 409 { 410 struct ttm_operation_ctx ctx = { 411 .interruptible = false, 412 .no_wait_gpu = false, 413 .force_alloc = true 414 }; 415 struct dma_fence *fence; 416 int ret; 417 unsigned i; 418 419 /* 420 * Can't use standard list traversal since we're unlocking. 421 */ 422 423 spin_lock(&bdev->lru_lock); 424 for (i = 0; i < TTM_MAX_BO_PRIORITY; ++i) { 425 while (!list_empty(&man->lru[i])) { 426 spin_unlock(&bdev->lru_lock); 427 ret = ttm_mem_evict_first(bdev, man, NULL, &ctx, 428 NULL); 429 if (ret) 430 return ret; 431 spin_lock(&bdev->lru_lock); 432 } 433 } 434 spin_unlock(&bdev->lru_lock); 435 436 spin_lock(&man->move_lock); 437 fence = dma_fence_get(man->move); 438 spin_unlock(&man->move_lock); 439 440 if (fence) { 441 ret = dma_fence_wait(fence, false); 442 dma_fence_put(fence); 443 if (ret) 444 return ret; 445 } 446 447 return 0; 448 } 449 EXPORT_SYMBOL(ttm_resource_manager_evict_all); 450 451 /** 452 * ttm_resource_manager_usage 453 * 454 * @man: A memory manager object. 455 * 456 * Return how many resources are currently used. 457 */ 458 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man) 459 { 460 uint64_t usage; 461 462 spin_lock(&man->bdev->lru_lock); 463 usage = man->usage; 464 spin_unlock(&man->bdev->lru_lock); 465 return usage; 466 } 467 EXPORT_SYMBOL(ttm_resource_manager_usage); 468 469 /** 470 * ttm_resource_manager_debug 471 * 472 * @man: manager type to dump. 473 * @p: printer to use for debug. 474 */ 475 void ttm_resource_manager_debug(struct ttm_resource_manager *man, 476 struct drm_printer *p) 477 { 478 drm_printf(p, " use_type: %d\n", man->use_type); 479 drm_printf(p, " use_tt: %d\n", man->use_tt); 480 drm_printf(p, " size: %llu\n", man->size); 481 drm_printf(p, " usage: %llu\n", ttm_resource_manager_usage(man)); 482 if (man->func->debug) 483 man->func->debug(man, p); 484 } 485 EXPORT_SYMBOL(ttm_resource_manager_debug); 486 487 /** 488 * ttm_resource_manager_first 489 * 490 * @man: resource manager to iterate over 491 * @cursor: cursor to record the position 492 * 493 * Returns the first resource from the resource manager. 494 */ 495 struct ttm_resource * 496 ttm_resource_manager_first(struct ttm_resource_manager *man, 497 struct ttm_resource_cursor *cursor) 498 { 499 lockdep_assert_held(&man->bdev->lru_lock); 500 501 cursor->priority = 0; 502 cursor->man = man; 503 cursor->cur = &man->lru[cursor->priority]; 504 return ttm_resource_manager_next(cursor); 505 } 506 507 /** 508 * ttm_resource_manager_next 509 * 510 * @cursor: cursor to record the position 511 * 512 * Return: the next resource from the resource manager. 513 */ 514 struct ttm_resource * 515 ttm_resource_manager_next(struct ttm_resource_cursor *cursor) 516 { 517 struct ttm_resource_manager *man = cursor->man; 518 struct ttm_lru_item *lru; 519 520 lockdep_assert_held(&man->bdev->lru_lock); 521 522 for (;;) { 523 lru = list_entry(cursor->cur, typeof(*lru), link); 524 list_for_each_entry_continue(lru, &man->lru[cursor->priority], link) { 525 if (ttm_lru_item_is_res(lru)) { 526 cursor->cur = &lru->link; 527 return ttm_lru_item_to_res(lru); 528 } 529 } 530 531 if (++cursor->priority >= TTM_MAX_BO_PRIORITY) 532 break; 533 534 cursor->cur = &man->lru[cursor->priority]; 535 } 536 537 return NULL; 538 } 539 540 /** 541 * ttm_lru_first_res_or_null() - Return the first resource on an lru list 542 * @head: The list head of the lru list. 543 * 544 * Return: Pointer to the first resource on the lru list or NULL if 545 * there is none. 546 */ 547 struct ttm_resource *ttm_lru_first_res_or_null(struct list_head *head) 548 { 549 struct ttm_lru_item *lru; 550 551 list_for_each_entry(lru, head, link) { 552 if (ttm_lru_item_is_res(lru)) 553 return ttm_lru_item_to_res(lru); 554 } 555 556 return NULL; 557 } 558 559 static void ttm_kmap_iter_iomap_map_local(struct ttm_kmap_iter *iter, 560 struct iosys_map *dmap, 561 pgoff_t i) 562 { 563 struct ttm_kmap_iter_iomap *iter_io = 564 container_of(iter, typeof(*iter_io), base); 565 void __iomem *addr; 566 567 retry: 568 while (i >= iter_io->cache.end) { 569 iter_io->cache.sg = iter_io->cache.sg ? 570 sg_next(iter_io->cache.sg) : iter_io->st->sgl; 571 iter_io->cache.i = iter_io->cache.end; 572 iter_io->cache.end += sg_dma_len(iter_io->cache.sg) >> 573 PAGE_SHIFT; 574 iter_io->cache.offs = sg_dma_address(iter_io->cache.sg) - 575 iter_io->start; 576 } 577 578 if (i < iter_io->cache.i) { 579 iter_io->cache.end = 0; 580 iter_io->cache.sg = NULL; 581 goto retry; 582 } 583 584 addr = io_mapping_map_local_wc(iter_io->iomap, iter_io->cache.offs + 585 (((resource_size_t)i - iter_io->cache.i) 586 << PAGE_SHIFT)); 587 iosys_map_set_vaddr_iomem(dmap, addr); 588 } 589 590 static void ttm_kmap_iter_iomap_unmap_local(struct ttm_kmap_iter *iter, 591 struct iosys_map *map) 592 { 593 io_mapping_unmap_local(map->vaddr_iomem); 594 } 595 596 static const struct ttm_kmap_iter_ops ttm_kmap_iter_io_ops = { 597 .map_local = ttm_kmap_iter_iomap_map_local, 598 .unmap_local = ttm_kmap_iter_iomap_unmap_local, 599 .maps_tt = false, 600 }; 601 602 /** 603 * ttm_kmap_iter_iomap_init - Initialize a struct ttm_kmap_iter_iomap 604 * @iter_io: The struct ttm_kmap_iter_iomap to initialize. 605 * @iomap: The struct io_mapping representing the underlying linear io_memory. 606 * @st: sg_table into @iomap, representing the memory of the struct 607 * ttm_resource. 608 * @start: Offset that needs to be subtracted from @st to make 609 * sg_dma_address(st->sgl) - @start == 0 for @iomap start. 610 * 611 * Return: Pointer to the embedded struct ttm_kmap_iter. 612 */ 613 struct ttm_kmap_iter * 614 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, 615 struct io_mapping *iomap, 616 struct sg_table *st, 617 resource_size_t start) 618 { 619 iter_io->base.ops = &ttm_kmap_iter_io_ops; 620 iter_io->iomap = iomap; 621 iter_io->st = st; 622 iter_io->start = start; 623 memset(&iter_io->cache, 0, sizeof(iter_io->cache)); 624 625 return &iter_io->base; 626 } 627 EXPORT_SYMBOL(ttm_kmap_iter_iomap_init); 628 629 /** 630 * DOC: Linear io iterator 631 * 632 * This code should die in the not too near future. Best would be if we could 633 * make io-mapping use memremap for all io memory, and have memremap 634 * implement a kmap_local functionality. We could then strip a huge amount of 635 * code. These linear io iterators are implemented to mimic old functionality, 636 * and they don't use kmap_local semantics at all internally. Rather ioremap or 637 * friends, and at least on 32-bit they add global TLB flushes and points 638 * of failure. 639 */ 640 641 static void ttm_kmap_iter_linear_io_map_local(struct ttm_kmap_iter *iter, 642 struct iosys_map *dmap, 643 pgoff_t i) 644 { 645 struct ttm_kmap_iter_linear_io *iter_io = 646 container_of(iter, typeof(*iter_io), base); 647 648 *dmap = iter_io->dmap; 649 iosys_map_incr(dmap, i * PAGE_SIZE); 650 } 651 652 static const struct ttm_kmap_iter_ops ttm_kmap_iter_linear_io_ops = { 653 .map_local = ttm_kmap_iter_linear_io_map_local, 654 .maps_tt = false, 655 }; 656 657 /** 658 * ttm_kmap_iter_linear_io_init - Initialize an iterator for linear io memory 659 * @iter_io: The iterator to initialize 660 * @bdev: The TTM device 661 * @mem: The ttm resource representing the iomap. 662 * 663 * This function is for internal TTM use only. It sets up a memcpy kmap iterator 664 * pointing at a linear chunk of io memory. 665 * 666 * Return: A pointer to the embedded struct ttm_kmap_iter or error pointer on 667 * failure. 668 */ 669 struct ttm_kmap_iter * 670 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, 671 struct ttm_device *bdev, 672 struct ttm_resource *mem) 673 { 674 int ret; 675 676 ret = ttm_mem_io_reserve(bdev, mem); 677 if (ret) 678 goto out_err; 679 if (!mem->bus.is_iomem) { 680 ret = -EINVAL; 681 goto out_io_free; 682 } 683 684 if (mem->bus.addr) { 685 iosys_map_set_vaddr(&iter_io->dmap, mem->bus.addr); 686 iter_io->needs_unmap = false; 687 } else { 688 iter_io->needs_unmap = true; 689 memset(&iter_io->dmap, 0, sizeof(iter_io->dmap)); 690 if (mem->bus.caching == ttm_write_combined) 691 iosys_map_set_vaddr_iomem(&iter_io->dmap, 692 ioremap_wc(mem->bus.offset, 693 mem->size)); 694 else if (mem->bus.caching == ttm_cached) 695 iosys_map_set_vaddr(&iter_io->dmap, 696 memremap(mem->bus.offset, mem->size, 697 MEMREMAP_WB | 698 MEMREMAP_WT | 699 MEMREMAP_WC)); 700 701 /* If uncached requested or if mapping cached or wc failed */ 702 if (iosys_map_is_null(&iter_io->dmap)) 703 iosys_map_set_vaddr_iomem(&iter_io->dmap, 704 ioremap(mem->bus.offset, 705 mem->size)); 706 707 if (iosys_map_is_null(&iter_io->dmap)) { 708 ret = -ENOMEM; 709 goto out_io_free; 710 } 711 } 712 713 iter_io->base.ops = &ttm_kmap_iter_linear_io_ops; 714 return &iter_io->base; 715 716 out_io_free: 717 ttm_mem_io_free(bdev, mem); 718 out_err: 719 return ERR_PTR(ret); 720 } 721 722 /** 723 * ttm_kmap_iter_linear_io_fini - Clean up an iterator for linear io memory 724 * @iter_io: The iterator to initialize 725 * @bdev: The TTM device 726 * @mem: The ttm resource representing the iomap. 727 * 728 * This function is for internal TTM use only. It cleans up a memcpy kmap 729 * iterator initialized by ttm_kmap_iter_linear_io_init. 730 */ 731 void 732 ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, 733 struct ttm_device *bdev, 734 struct ttm_resource *mem) 735 { 736 if (iter_io->needs_unmap && iosys_map_is_set(&iter_io->dmap)) { 737 if (iter_io->dmap.is_iomem) 738 iounmap(iter_io->dmap.vaddr_iomem); 739 else 740 memunmap(iter_io->dmap.vaddr); 741 } 742 743 ttm_mem_io_free(bdev, mem); 744 } 745 746 #if defined(CONFIG_DEBUG_FS) 747 748 static int ttm_resource_manager_show(struct seq_file *m, void *unused) 749 { 750 struct ttm_resource_manager *man = 751 (struct ttm_resource_manager *)m->private; 752 struct drm_printer p = drm_seq_file_printer(m); 753 ttm_resource_manager_debug(man, &p); 754 return 0; 755 } 756 DEFINE_SHOW_ATTRIBUTE(ttm_resource_manager); 757 758 #endif 759 760 /** 761 * ttm_resource_manager_create_debugfs - Create debugfs entry for specified 762 * resource manager. 763 * @man: The TTM resource manager for which the debugfs stats file be creates 764 * @parent: debugfs directory in which the file will reside 765 * @name: The filename to create. 766 * 767 * This function setups up a debugfs file that can be used to look 768 * at debug statistics of the specified ttm_resource_manager. 769 */ 770 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, 771 struct dentry * parent, 772 const char *name) 773 { 774 #if defined(CONFIG_DEBUG_FS) 775 debugfs_create_file(name, 0444, parent, man, &ttm_resource_manager_fops); 776 #endif 777 } 778 EXPORT_SYMBOL(ttm_resource_manager_create_debugfs); 779