1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021-2022 Intel Corporation 4 * Copyright (C) 2021-2022 Red Hat 5 */ 6 7 #include <linux/cgroup_dmem.h> 8 9 #include <drm/drm_managed.h> 10 #include <drm/drm_drv.h> 11 #include <drm/drm_buddy.h> 12 13 #include <drm/ttm/ttm_placement.h> 14 #include <drm/ttm/ttm_range_manager.h> 15 16 #include "xe_bo.h" 17 #include "xe_device.h" 18 #include "xe_pm.h" 19 #include "xe_res_cursor.h" 20 #include "xe_ttm_vram_mgr.h" 21 #include "xe_vram_types.h" 22 23 static inline struct gpu_buddy_block * 24 xe_ttm_vram_mgr_first_block(struct list_head *list) 25 { 26 return list_first_entry_or_null(list, struct gpu_buddy_block, link); 27 } 28 29 static inline bool xe_is_vram_mgr_blocks_contiguous(struct gpu_buddy *mm, 30 struct list_head *head) 31 { 32 struct gpu_buddy_block *block; 33 u64 start, size; 34 35 block = xe_ttm_vram_mgr_first_block(head); 36 if (!block) 37 return false; 38 39 while (head != block->link.next) { 40 start = gpu_buddy_block_offset(block); 41 size = gpu_buddy_block_size(mm, block); 42 43 block = list_entry(block->link.next, struct gpu_buddy_block, 44 link); 45 if (start + size != gpu_buddy_block_offset(block)) 46 return false; 47 } 48 49 return true; 50 } 51 52 static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, 53 struct ttm_buffer_object *tbo, 54 const struct ttm_place *place, 55 struct ttm_resource **res) 56 { 57 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 58 struct xe_ttm_vram_mgr_resource *vres; 59 struct gpu_buddy *mm = &mgr->mm; 60 u64 size, min_page_size; 61 unsigned long lpfn; 62 int err; 63 64 lpfn = place->lpfn; 65 if (!lpfn || lpfn > man->size >> PAGE_SHIFT) 66 lpfn = man->size >> PAGE_SHIFT; 67 68 if (tbo->base.size >> PAGE_SHIFT > (lpfn - place->fpfn)) 69 return -E2BIG; /* don't trigger eviction for the impossible */ 70 71 vres = kzalloc_obj(*vres); 72 if (!vres) 73 return -ENOMEM; 74 75 ttm_resource_init(tbo, place, &vres->base); 76 77 /* bail out quickly if there's likely not enough VRAM for this BO */ 78 if (ttm_resource_manager_usage(man) > man->size) { 79 err = -ENOSPC; 80 goto error_fini; 81 } 82 83 INIT_LIST_HEAD(&vres->blocks); 84 85 if (place->flags & TTM_PL_FLAG_TOPDOWN) 86 vres->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION; 87 88 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) 89 vres->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION; 90 91 if (place->fpfn || lpfn != man->size >> PAGE_SHIFT) 92 vres->flags |= GPU_BUDDY_RANGE_ALLOCATION; 93 94 if (WARN_ON(!vres->base.size)) { 95 err = -EINVAL; 96 goto error_fini; 97 } 98 size = vres->base.size; 99 100 min_page_size = mgr->default_page_size; 101 if (tbo->page_alignment) 102 min_page_size = (u64)tbo->page_alignment << PAGE_SHIFT; 103 104 if (WARN_ON(min_page_size < mm->chunk_size)) { 105 err = -EINVAL; 106 goto error_fini; 107 } 108 109 if (WARN_ON(!IS_ALIGNED(size, min_page_size))) { 110 err = -EINVAL; 111 goto error_fini; 112 } 113 114 mutex_lock(&mgr->lock); 115 if (lpfn <= mgr->visible_size >> PAGE_SHIFT && size > mgr->visible_avail) { 116 err = -ENOSPC; 117 goto error_unlock; 118 } 119 120 err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, 121 (u64)lpfn << PAGE_SHIFT, size, 122 min_page_size, &vres->blocks, vres->flags); 123 if (err) 124 goto error_unlock; 125 126 if (lpfn <= mgr->visible_size >> PAGE_SHIFT) { 127 vres->used_visible_size = size; 128 } else { 129 struct gpu_buddy_block *block; 130 131 list_for_each_entry(block, &vres->blocks, link) { 132 u64 start = gpu_buddy_block_offset(block); 133 134 if (start < mgr->visible_size) { 135 u64 end = start + gpu_buddy_block_size(mm, block); 136 137 vres->used_visible_size += 138 min(end, mgr->visible_size) - start; 139 } 140 } 141 } 142 143 mgr->visible_avail -= vres->used_visible_size; 144 mutex_unlock(&mgr->lock); 145 146 if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) && 147 xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks)) 148 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; 149 150 /* 151 * For some kernel objects we still rely on the start when io mapping 152 * the object. 153 */ 154 if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) { 155 struct gpu_buddy_block *block = list_first_entry(&vres->blocks, 156 typeof(*block), 157 link); 158 159 vres->base.start = gpu_buddy_block_offset(block) >> PAGE_SHIFT; 160 } else { 161 vres->base.start = XE_BO_INVALID_OFFSET; 162 } 163 164 *res = &vres->base; 165 return 0; 166 error_unlock: 167 mutex_unlock(&mgr->lock); 168 error_fini: 169 ttm_resource_fini(man, &vres->base); 170 kfree(vres); 171 172 return err; 173 } 174 175 static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, 176 struct ttm_resource *res) 177 { 178 struct xe_ttm_vram_mgr_resource *vres = 179 to_xe_ttm_vram_mgr_resource(res); 180 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 181 struct gpu_buddy *mm = &mgr->mm; 182 183 mutex_lock(&mgr->lock); 184 gpu_buddy_free_list(mm, &vres->blocks, 0); 185 mgr->visible_avail += vres->used_visible_size; 186 mutex_unlock(&mgr->lock); 187 188 ttm_resource_fini(man, res); 189 190 kfree(vres); 191 } 192 193 static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man, 194 struct drm_printer *printer) 195 { 196 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 197 struct gpu_buddy *mm = &mgr->mm; 198 199 mutex_lock(&mgr->lock); 200 drm_printf(printer, "default_page_size: %lluKiB\n", 201 mgr->default_page_size >> 10); 202 drm_printf(printer, "visible_avail: %lluMiB\n", 203 (u64)mgr->visible_avail >> 20); 204 drm_printf(printer, "visible_size: %lluMiB\n", 205 (u64)mgr->visible_size >> 20); 206 207 drm_buddy_print(mm, printer); 208 mutex_unlock(&mgr->lock); 209 drm_printf(printer, "man size:%llu\n", man->size); 210 } 211 212 static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man, 213 struct ttm_resource *res, 214 const struct ttm_place *place, 215 size_t size) 216 { 217 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 218 struct xe_ttm_vram_mgr_resource *vres = 219 to_xe_ttm_vram_mgr_resource(res); 220 struct gpu_buddy *mm = &mgr->mm; 221 struct gpu_buddy_block *block; 222 223 if (!place->fpfn && !place->lpfn) 224 return true; 225 226 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT) 227 return vres->used_visible_size > 0; 228 229 list_for_each_entry(block, &vres->blocks, link) { 230 unsigned long fpfn = 231 gpu_buddy_block_offset(block) >> PAGE_SHIFT; 232 unsigned long lpfn = fpfn + 233 (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); 234 235 if (place->fpfn < lpfn && place->lpfn > fpfn) 236 return true; 237 } 238 239 return false; 240 } 241 242 static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man, 243 struct ttm_resource *res, 244 const struct ttm_place *place, 245 size_t size) 246 { 247 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 248 struct xe_ttm_vram_mgr_resource *vres = 249 to_xe_ttm_vram_mgr_resource(res); 250 struct gpu_buddy *mm = &mgr->mm; 251 struct gpu_buddy_block *block; 252 253 if (!place->fpfn && !place->lpfn) 254 return true; 255 256 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT) 257 return vres->used_visible_size == size; 258 259 list_for_each_entry(block, &vres->blocks, link) { 260 unsigned long fpfn = 261 gpu_buddy_block_offset(block) >> PAGE_SHIFT; 262 unsigned long lpfn = fpfn + 263 (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); 264 265 if (fpfn < place->fpfn || lpfn > place->lpfn) 266 return false; 267 } 268 269 return true; 270 } 271 272 static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = { 273 .alloc = xe_ttm_vram_mgr_new, 274 .free = xe_ttm_vram_mgr_del, 275 .intersects = xe_ttm_vram_mgr_intersects, 276 .compatible = xe_ttm_vram_mgr_compatible, 277 .debug = xe_ttm_vram_mgr_debug 278 }; 279 280 static const struct dmem_cgroup_ops xe_ttm_vram_mgr_dmem_ops; 281 282 static int xe_ttm_vram_mgr_dmem_reclaim(struct dmem_cgroup_pool_state *pool, 283 u64 target_bytes, void *priv) 284 { 285 struct ttm_resource_manager *man = priv; 286 struct xe_device *xe = ttm_to_xe_device(man->bdev); 287 int ret, idx; 288 289 if (!drm_dev_enter(&xe->drm, &idx)) 290 return -ENODEV; 291 292 { 293 ACQUIRE(xe_pm_runtime_ioctl, pm)(xe); 294 295 ret = ACQUIRE_ERR(xe_pm_runtime_ioctl, &pm); 296 if (ret >= 0) 297 ret = ttm_resource_manager_dmem_reclaim(pool, target_bytes, priv); 298 } 299 300 drm_dev_exit(idx); 301 return ret; 302 } 303 304 static const struct dmem_cgroup_ops xe_ttm_vram_mgr_dmem_ops = { 305 .reclaim = xe_ttm_vram_mgr_dmem_reclaim, 306 }; 307 308 static void xe_ttm_vram_mgr_set_unused(struct drm_device *dev, void *arg) 309 { 310 struct ttm_resource_manager *man = arg; 311 312 ttm_resource_manager_set_used(man, false); 313 } 314 315 static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg) 316 { 317 struct xe_device *xe = to_xe_device(dev); 318 struct xe_ttm_vram_mgr *mgr = arg; 319 struct ttm_resource_manager *man = &mgr->manager; 320 321 if (ttm_resource_manager_evict_all(&xe->ttm, man)) 322 return; 323 324 WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size); 325 326 gpu_buddy_fini(&mgr->mm); 327 328 ttm_resource_manager_cleanup(&mgr->manager); 329 330 ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL); 331 } 332 333 int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, 334 u32 mem_type, u64 size, u64 io_size, 335 u64 default_page_size) 336 { 337 struct ttm_resource_manager *man = &mgr->manager; 338 struct dmem_cgroup_region *cg; 339 const char *name; 340 int err; 341 342 man->func = &xe_ttm_vram_mgr_func; 343 mgr->mem_type = mem_type; 344 err = drmm_mutex_init(&xe->drm, &mgr->lock); 345 if (err) 346 return err; 347 mgr->default_page_size = default_page_size; 348 mgr->visible_size = io_size; 349 mgr->visible_avail = io_size; 350 351 ttm_resource_manager_init(man, &xe->ttm, size); 352 err = gpu_buddy_init(&mgr->mm, man->size, default_page_size); 353 if (err) 354 return err; 355 356 gpu_buddy_driver_set_lock(&mgr->mm, &mgr->lock); 357 ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager); 358 359 /* 360 * Register the fini action before the cgroup region so that devres 361 * LIFO teardown runs unregister_region before manager teardown 362 * (draining any in-flight reclaim callbacks) and the manager fini second. 363 */ 364 err = drmm_add_action_or_reset(&xe->drm, xe_ttm_vram_mgr_fini, mgr); 365 if (err) 366 return err; 367 368 name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; 369 cg = drmm_cgroup_register_region(&xe->drm, name, 370 &(struct dmem_cgroup_init){ 371 .size = size, 372 .ops = &xe_ttm_vram_mgr_dmem_ops, 373 .reclaim_priv = man, 374 }); 375 if (IS_ERR(cg)) 376 return PTR_ERR(cg); 377 378 ttm_resource_manager_set_dmem_region(man, cg); 379 ttm_resource_manager_set_used(man, true); 380 381 return drmm_add_action_or_reset(&xe->drm, xe_ttm_vram_mgr_set_unused, man); 382 } 383 384 /** 385 * xe_ttm_vram_mgr_init - initialize TTM VRAM region 386 * @xe: pointer to Xe device 387 * @vram: pointer to xe_vram_region that contains the memory region attributes 388 * 389 * Initialize the Xe TTM for given @vram region using the given parameters. 390 * 391 * Returns 0 for success, negative error code otherwise. 392 */ 393 int xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_vram_region *vram) 394 { 395 return __xe_ttm_vram_mgr_init(xe, &vram->ttm, vram->placement, 396 xe_vram_region_usable_size(vram), 397 xe_vram_region_io_size(vram), 398 PAGE_SIZE); 399 } 400 401 int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, 402 struct ttm_resource *res, 403 u64 offset, u64 length, 404 struct device *dev, 405 enum dma_data_direction dir, 406 struct sg_table **sgt) 407 { 408 struct xe_tile *tile = &xe->tiles[res->mem_type - XE_PL_VRAM0]; 409 struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res); 410 struct xe_res_cursor cursor; 411 struct scatterlist *sg; 412 int num_entries = 0; 413 int i, r; 414 415 if (vres->used_visible_size < res->size) 416 return -EOPNOTSUPP; 417 418 *sgt = kmalloc_obj(**sgt); 419 if (!*sgt) 420 return -ENOMEM; 421 422 /* Determine the number of GPU_BUDDY blocks to export */ 423 xe_res_first(res, offset, length, &cursor); 424 while (cursor.remaining) { 425 num_entries++; 426 /* Limit maximum size to 2GiB due to SG table limitations. */ 427 xe_res_next(&cursor, min_t(u64, cursor.size, SZ_2G)); 428 } 429 430 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); 431 if (r) 432 goto error_free; 433 434 /* Initialize scatterlist nodes of sg_table */ 435 for_each_sgtable_sg((*sgt), sg, i) 436 sg->length = 0; 437 438 /* 439 * Walk down GPU_BUDDY blocks to populate scatterlist nodes 440 * @note: Use iterator api to get first the GPU_BUDDY block 441 * and the number of bytes from it. Access the following 442 * GPU_BUDDY block(s) if more buffer needs to exported 443 */ 444 xe_res_first(res, offset, length, &cursor); 445 for_each_sgtable_sg((*sgt), sg, i) { 446 phys_addr_t phys = cursor.start + xe_vram_region_io_start(tile->mem.vram); 447 size_t size = min_t(u64, cursor.size, SZ_2G); 448 dma_addr_t addr; 449 450 addr = dma_map_resource(dev, phys, size, dir, 451 DMA_ATTR_SKIP_CPU_SYNC); 452 r = dma_mapping_error(dev, addr); 453 if (r) 454 goto error_unmap; 455 456 sg_set_page(sg, NULL, size, 0); 457 sg_dma_address(sg) = addr; 458 sg_dma_len(sg) = size; 459 460 xe_res_next(&cursor, size); 461 } 462 463 return 0; 464 465 error_unmap: 466 for_each_sgtable_sg((*sgt), sg, i) { 467 if (!sg->length) 468 continue; 469 470 dma_unmap_resource(dev, sg->dma_address, 471 sg->length, dir, 472 DMA_ATTR_SKIP_CPU_SYNC); 473 } 474 sg_free_table(*sgt); 475 476 error_free: 477 kfree(*sgt); 478 return r; 479 } 480 481 void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir, 482 struct sg_table *sgt) 483 { 484 struct scatterlist *sg; 485 int i; 486 487 for_each_sgtable_sg(sgt, sg, i) 488 dma_unmap_resource(dev, sg->dma_address, 489 sg->length, dir, 490 DMA_ATTR_SKIP_CPU_SYNC); 491 sg_free_table(sgt); 492 kfree(sgt); 493 } 494 495 u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man) 496 { 497 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 498 499 return mgr->visible_size; 500 } 501 502 void xe_ttm_vram_get_used(struct ttm_resource_manager *man, 503 u64 *used, u64 *used_visible) 504 { 505 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 506 507 mutex_lock(&mgr->lock); 508 *used = mgr->mm.size - mgr->mm.avail; 509 *used_visible = mgr->visible_size - mgr->visible_avail; 510 mutex_unlock(&mgr->lock); 511 } 512 513 u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man) 514 { 515 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 516 u64 avail; 517 518 mutex_lock(&mgr->lock); 519 avail = mgr->mm.avail; 520 mutex_unlock(&mgr->lock); 521 522 return avail; 523 } 524