1 // SPDX-License-Identifier: GPL-2.0+ 2 /* 3 * Copyright (C) 2023 Loongson Technology Corporation Limited 4 */ 5 6 #include <drm/drm_drv.h> 7 #include <drm/drm_file.h> 8 #include <drm/drm_gem.h> 9 #include <drm/drm_managed.h> 10 #include <drm/drm_prime.h> 11 12 #include "lsdc_drv.h" 13 #include "lsdc_ttm.h" 14 15 const char *lsdc_mem_type_to_str(uint32_t mem_type) 16 { 17 switch (mem_type) { 18 case TTM_PL_VRAM: 19 return "VRAM"; 20 case TTM_PL_TT: 21 return "GTT"; 22 case TTM_PL_SYSTEM: 23 return "SYSTEM"; 24 default: 25 break; 26 } 27 28 return "Unknown"; 29 } 30 31 const char *lsdc_domain_to_str(u32 domain) 32 { 33 switch (domain) { 34 case LSDC_GEM_DOMAIN_VRAM: 35 return "VRAM"; 36 case LSDC_GEM_DOMAIN_GTT: 37 return "GTT"; 38 case LSDC_GEM_DOMAIN_SYSTEM: 39 return "SYSTEM"; 40 default: 41 break; 42 } 43 44 return "Unknown"; 45 } 46 47 static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain) 48 { 49 u32 c = 0; 50 u32 pflags = 0; 51 u32 i; 52 53 if (lbo->tbo.base.size <= PAGE_SIZE) 54 pflags |= TTM_PL_FLAG_TOPDOWN; 55 56 lbo->placement.placement = lbo->placements; 57 58 if (domain & LSDC_GEM_DOMAIN_VRAM) { 59 lbo->placements[c].mem_type = TTM_PL_VRAM; 60 lbo->placements[c++].flags = pflags; 61 } 62 63 if (domain & LSDC_GEM_DOMAIN_GTT) { 64 lbo->placements[c].mem_type = TTM_PL_TT; 65 lbo->placements[c++].flags = pflags; 66 } 67 68 if (domain & LSDC_GEM_DOMAIN_SYSTEM) { 69 lbo->placements[c].mem_type = TTM_PL_SYSTEM; 70 lbo->placements[c++].flags = 0; 71 } 72 73 if (!c) { 74 lbo->placements[c].mem_type = TTM_PL_SYSTEM; 75 lbo->placements[c++].flags = 0; 76 } 77 78 lbo->placement.num_placement = c; 79 80 for (i = 0; i < c; ++i) { 81 lbo->placements[i].fpfn = 0; 82 lbo->placements[i].lpfn = 0; 83 } 84 } 85 86 static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt) 87 { 88 ttm_tt_fini(tt); 89 kfree(tt); 90 } 91 92 static struct ttm_tt * 93 lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags) 94 { 95 struct ttm_tt *tt; 96 int ret; 97 98 tt = kzalloc(sizeof(*tt), GFP_KERNEL); 99 if (!tt) 100 return NULL; 101 102 ret = ttm_sg_tt_init(tt, tbo, page_flags, ttm_cached); 103 if (ret < 0) { 104 kfree(tt); 105 return NULL; 106 } 107 108 return tt; 109 } 110 111 static int lsdc_ttm_tt_populate(struct ttm_device *bdev, 112 struct ttm_tt *ttm, 113 struct ttm_operation_ctx *ctx) 114 { 115 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 116 117 if (slave && ttm->sg) { 118 drm_prime_sg_to_dma_addr_array(ttm->sg, 119 ttm->dma_address, 120 ttm->num_pages); 121 122 return 0; 123 } 124 125 return ttm_pool_alloc(&bdev->pool, ttm, ctx); 126 } 127 128 static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev, 129 struct ttm_tt *ttm) 130 { 131 bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL); 132 133 if (slave) 134 return; 135 136 return ttm_pool_free(&bdev->pool, ttm); 137 } 138 139 static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo, 140 struct ttm_placement *tplacement) 141 { 142 struct ttm_resource *resource = tbo->resource; 143 struct lsdc_bo *lbo = to_lsdc_bo(tbo); 144 145 switch (resource->mem_type) { 146 case TTM_PL_VRAM: 147 lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT); 148 break; 149 case TTM_PL_TT: 150 default: 151 lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM); 152 break; 153 } 154 155 *tplacement = lbo->placement; 156 } 157 158 static int lsdc_bo_move(struct ttm_buffer_object *tbo, 159 bool evict, 160 struct ttm_operation_ctx *ctx, 161 struct ttm_resource *new_mem, 162 struct ttm_place *hop) 163 { 164 struct drm_device *ddev = tbo->base.dev; 165 struct ttm_resource *old_mem = tbo->resource; 166 struct lsdc_bo *lbo = to_lsdc_bo(tbo); 167 int ret; 168 169 if (unlikely(tbo->pin_count > 0)) { 170 drm_warn(ddev, "Can't move a pinned BO\n"); 171 return -EINVAL; 172 } 173 174 ret = ttm_bo_wait_ctx(tbo, ctx); 175 if (ret) 176 return ret; 177 178 if (!old_mem) { 179 drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n", 180 lbo, lsdc_mem_type_to_str(new_mem->mem_type), 181 lsdc_bo_size(lbo)); 182 ttm_bo_move_null(tbo, new_mem); 183 return 0; 184 } 185 186 if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) { 187 ttm_bo_move_null(tbo, new_mem); 188 drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n", 189 lbo, lsdc_bo_size(lbo)); 190 return 0; 191 } 192 193 if (old_mem->mem_type == TTM_PL_SYSTEM && 194 new_mem->mem_type == TTM_PL_TT) { 195 drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n", 196 lbo, lsdc_bo_size(lbo)); 197 ttm_bo_move_null(tbo, new_mem); 198 return 0; 199 } 200 201 if (old_mem->mem_type == TTM_PL_TT && 202 new_mem->mem_type == TTM_PL_SYSTEM) { 203 drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n", 204 lbo, lsdc_bo_size(lbo)); 205 ttm_resource_free(tbo, &tbo->resource); 206 ttm_bo_assign_mem(tbo, new_mem); 207 return 0; 208 } 209 210 drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n", 211 lbo, 212 lsdc_mem_type_to_str(old_mem->mem_type), 213 lsdc_mem_type_to_str(new_mem->mem_type), 214 lsdc_bo_size(lbo)); 215 216 return ttm_bo_move_memcpy(tbo, ctx, new_mem); 217 } 218 219 static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev, 220 struct ttm_resource *mem) 221 { 222 struct lsdc_device *ldev = tdev_to_ldev(bdev); 223 224 switch (mem->mem_type) { 225 case TTM_PL_SYSTEM: 226 break; 227 case TTM_PL_TT: 228 break; 229 case TTM_PL_VRAM: 230 mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base; 231 mem->bus.is_iomem = true; 232 mem->bus.caching = ttm_write_combined; 233 break; 234 default: 235 return -EINVAL; 236 } 237 238 return 0; 239 } 240 241 static struct ttm_device_funcs lsdc_bo_driver = { 242 .ttm_tt_create = lsdc_ttm_tt_create, 243 .ttm_tt_populate = lsdc_ttm_tt_populate, 244 .ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate, 245 .ttm_tt_destroy = lsdc_ttm_tt_destroy, 246 .eviction_valuable = ttm_bo_eviction_valuable, 247 .evict_flags = lsdc_bo_evict_flags, 248 .move = lsdc_bo_move, 249 .io_mem_reserve = lsdc_bo_reserve_io_mem, 250 }; 251 252 u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo) 253 { 254 struct ttm_buffer_object *tbo = &lbo->tbo; 255 struct drm_device *ddev = tbo->base.dev; 256 struct ttm_resource *resource = tbo->resource; 257 258 if (unlikely(!tbo->pin_count)) { 259 drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n"); 260 return 0; 261 } 262 263 if (unlikely(resource->mem_type == TTM_PL_SYSTEM)) 264 return 0; 265 266 return resource->start << PAGE_SHIFT; 267 } 268 269 size_t lsdc_bo_size(struct lsdc_bo *lbo) 270 { 271 struct ttm_buffer_object *tbo = &lbo->tbo; 272 273 return tbo->base.size; 274 } 275 276 int lsdc_bo_reserve(struct lsdc_bo *lbo) 277 { 278 return ttm_bo_reserve(&lbo->tbo, true, false, NULL); 279 } 280 281 void lsdc_bo_unreserve(struct lsdc_bo *lbo) 282 { 283 return ttm_bo_unreserve(&lbo->tbo); 284 } 285 286 int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr) 287 { 288 struct ttm_operation_ctx ctx = { false, false }; 289 struct ttm_buffer_object *tbo = &lbo->tbo; 290 struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev); 291 int ret; 292 293 if (tbo->pin_count) 294 goto bo_pinned; 295 296 if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM) 297 return -EINVAL; 298 299 if (domain) 300 lsdc_bo_set_placement(lbo, domain); 301 302 ret = ttm_bo_validate(tbo, &lbo->placement, &ctx); 303 if (unlikely(ret)) { 304 drm_err(&ldev->base, "%p validate failed: %d\n", lbo, ret); 305 return ret; 306 } 307 308 if (domain == LSDC_GEM_DOMAIN_VRAM) 309 ldev->vram_pinned_size += lsdc_bo_size(lbo); 310 else if (domain == LSDC_GEM_DOMAIN_GTT) 311 ldev->gtt_pinned_size += lsdc_bo_size(lbo); 312 313 bo_pinned: 314 ttm_bo_pin(tbo); 315 316 if (gpu_addr) 317 *gpu_addr = lsdc_bo_gpu_offset(lbo); 318 319 return 0; 320 } 321 322 void lsdc_bo_unpin(struct lsdc_bo *lbo) 323 { 324 struct ttm_buffer_object *tbo = &lbo->tbo; 325 struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev); 326 327 if (unlikely(!tbo->pin_count)) { 328 drm_dbg(&ldev->base, "%p unpin is not necessary\n", lbo); 329 return; 330 } 331 332 ttm_bo_unpin(tbo); 333 334 if (!tbo->pin_count) { 335 if (tbo->resource->mem_type == TTM_PL_VRAM) 336 ldev->vram_pinned_size -= lsdc_bo_size(lbo); 337 else if (tbo->resource->mem_type == TTM_PL_TT) 338 ldev->gtt_pinned_size -= lsdc_bo_size(lbo); 339 } 340 } 341 342 void lsdc_bo_ref(struct lsdc_bo *lbo) 343 { 344 drm_gem_object_get(&lbo->tbo.base); 345 } 346 347 void lsdc_bo_unref(struct lsdc_bo *lbo) 348 { 349 drm_gem_object_put(&lbo->tbo.base); 350 } 351 352 int lsdc_bo_kmap(struct lsdc_bo *lbo) 353 { 354 struct ttm_buffer_object *tbo = &lbo->tbo; 355 struct drm_gem_object *gem = &tbo->base; 356 struct drm_device *ddev = gem->dev; 357 long ret; 358 int err; 359 360 ret = dma_resv_wait_timeout(gem->resv, DMA_RESV_USAGE_KERNEL, false, 361 MAX_SCHEDULE_TIMEOUT); 362 if (ret < 0) { 363 drm_warn(ddev, "wait fence timeout\n"); 364 return ret; 365 } 366 367 if (lbo->kptr) 368 return 0; 369 370 err = ttm_bo_kmap(tbo, 0, PFN_UP(lsdc_bo_size(lbo)), &lbo->kmap); 371 if (err) { 372 drm_err(ddev, "kmap %p failed: %d\n", lbo, err); 373 return err; 374 } 375 376 lbo->kptr = ttm_kmap_obj_virtual(&lbo->kmap, &lbo->is_iomem); 377 378 return 0; 379 } 380 381 void lsdc_bo_kunmap(struct lsdc_bo *lbo) 382 { 383 if (!lbo->kptr) 384 return; 385 386 lbo->kptr = NULL; 387 ttm_bo_kunmap(&lbo->kmap); 388 } 389 390 void lsdc_bo_clear(struct lsdc_bo *lbo) 391 { 392 lsdc_bo_kmap(lbo); 393 394 if (lbo->is_iomem) 395 memset_io((void __iomem *)lbo->kptr, 0, lbo->size); 396 else 397 memset(lbo->kptr, 0, lbo->size); 398 399 lsdc_bo_kunmap(lbo); 400 } 401 402 int lsdc_bo_evict_vram(struct drm_device *ddev) 403 { 404 struct lsdc_device *ldev = to_lsdc(ddev); 405 struct ttm_device *bdev = &ldev->bdev; 406 struct ttm_resource_manager *man; 407 408 man = ttm_manager_type(bdev, TTM_PL_VRAM); 409 if (unlikely(!man)) 410 return 0; 411 412 return ttm_resource_manager_evict_all(bdev, man); 413 } 414 415 static void lsdc_bo_destroy(struct ttm_buffer_object *tbo) 416 { 417 struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev); 418 struct lsdc_bo *lbo = to_lsdc_bo(tbo); 419 420 mutex_lock(&ldev->gem.mutex); 421 list_del_init(&lbo->list); 422 mutex_unlock(&ldev->gem.mutex); 423 424 drm_gem_object_release(&tbo->base); 425 426 kfree(lbo); 427 } 428 429 struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev, 430 u32 domain, 431 size_t size, 432 bool kernel, 433 struct sg_table *sg, 434 struct dma_resv *resv) 435 { 436 struct lsdc_device *ldev = to_lsdc(ddev); 437 struct ttm_device *bdev = &ldev->bdev; 438 struct ttm_buffer_object *tbo; 439 struct lsdc_bo *lbo; 440 enum ttm_bo_type bo_type; 441 int ret; 442 443 lbo = kzalloc(sizeof(*lbo), GFP_KERNEL); 444 if (!lbo) 445 return ERR_PTR(-ENOMEM); 446 447 INIT_LIST_HEAD(&lbo->list); 448 449 lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM | 450 LSDC_GEM_DOMAIN_GTT | 451 LSDC_GEM_DOMAIN_SYSTEM); 452 453 tbo = &lbo->tbo; 454 455 size = ALIGN(size, PAGE_SIZE); 456 457 ret = drm_gem_object_init(ddev, &tbo->base, size); 458 if (ret) { 459 kfree(lbo); 460 return ERR_PTR(ret); 461 } 462 463 tbo->bdev = bdev; 464 465 if (kernel) 466 bo_type = ttm_bo_type_kernel; 467 else if (sg) 468 bo_type = ttm_bo_type_sg; 469 else 470 bo_type = ttm_bo_type_device; 471 472 lsdc_bo_set_placement(lbo, domain); 473 lbo->size = size; 474 475 ret = ttm_bo_init_validate(bdev, tbo, bo_type, &lbo->placement, 0, 476 false, sg, resv, lsdc_bo_destroy); 477 if (ret) { 478 kfree(lbo); 479 return ERR_PTR(ret); 480 } 481 482 return lbo; 483 } 484 485 struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev, 486 u32 domain, 487 size_t size) 488 { 489 struct lsdc_bo *lbo; 490 int ret; 491 492 lbo = lsdc_bo_create(ddev, domain, size, true, NULL, NULL); 493 if (IS_ERR(lbo)) 494 return ERR_CAST(lbo); 495 496 ret = lsdc_bo_reserve(lbo); 497 if (unlikely(ret)) { 498 lsdc_bo_unref(lbo); 499 return ERR_PTR(ret); 500 } 501 502 ret = lsdc_bo_pin(lbo, domain, NULL); 503 lsdc_bo_unreserve(lbo); 504 if (unlikely(ret)) { 505 lsdc_bo_unref(lbo); 506 return ERR_PTR(ret); 507 } 508 509 return lbo; 510 } 511 512 void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo) 513 { 514 int ret; 515 516 ret = lsdc_bo_reserve(lbo); 517 if (unlikely(ret)) 518 return; 519 520 lsdc_bo_unpin(lbo); 521 lsdc_bo_unreserve(lbo); 522 523 lsdc_bo_unref(lbo); 524 } 525 526 static void lsdc_ttm_fini(struct drm_device *ddev, void *data) 527 { 528 struct lsdc_device *ldev = (struct lsdc_device *)data; 529 530 ttm_range_man_fini(&ldev->bdev, TTM_PL_VRAM); 531 ttm_range_man_fini(&ldev->bdev, TTM_PL_TT); 532 533 ttm_device_fini(&ldev->bdev); 534 535 drm_dbg(ddev, "ttm finished\n"); 536 } 537 538 int lsdc_ttm_init(struct lsdc_device *ldev) 539 { 540 struct drm_device *ddev = &ldev->base; 541 unsigned long num_vram_pages; 542 unsigned long num_gtt_pages; 543 int ret; 544 545 ret = ttm_device_init(&ldev->bdev, &lsdc_bo_driver, ddev->dev, 546 ddev->anon_inode->i_mapping, 547 ddev->vma_offset_manager, false, true); 548 if (ret) 549 return ret; 550 551 num_vram_pages = ldev->vram_size >> PAGE_SHIFT; 552 553 ret = ttm_range_man_init(&ldev->bdev, TTM_PL_VRAM, false, num_vram_pages); 554 if (unlikely(ret)) 555 return ret; 556 557 drm_info(ddev, "VRAM: %lu pages ready\n", num_vram_pages); 558 559 /* 512M is far enough for us now */ 560 ldev->gtt_size = 512 << 20; 561 562 num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT; 563 564 ret = ttm_range_man_init(&ldev->bdev, TTM_PL_TT, true, num_gtt_pages); 565 if (unlikely(ret)) 566 return ret; 567 568 drm_info(ddev, "GTT: %lu pages ready\n", num_gtt_pages); 569 570 return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev); 571 } 572 573 void lsdc_ttm_debugfs_init(struct lsdc_device *ldev) 574 { 575 struct ttm_device *bdev = &ldev->bdev; 576 struct drm_device *ddev = &ldev->base; 577 struct drm_minor *minor = ddev->primary; 578 struct dentry *root = minor->debugfs_root; 579 struct ttm_resource_manager *vram_man; 580 struct ttm_resource_manager *gtt_man; 581 582 vram_man = ttm_manager_type(bdev, TTM_PL_VRAM); 583 gtt_man = ttm_manager_type(bdev, TTM_PL_TT); 584 585 ttm_resource_manager_create_debugfs(vram_man, root, "vram_mm"); 586 ttm_resource_manager_create_debugfs(gtt_man, root, "gtt_mm"); 587 } 588