1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021-2022 Intel Corporation 4 * Copyright (C) 2021-2022 Red Hat 5 */ 6 7 #include <drm/drm_managed.h> 8 #include <drm/drm_drv.h> 9 #include <drm/drm_buddy.h> 10 11 #include <drm/ttm/ttm_placement.h> 12 #include <drm/ttm/ttm_range_manager.h> 13 14 #include "xe_bo.h" 15 #include "xe_device.h" 16 #include "xe_res_cursor.h" 17 #include "xe_ttm_vram_mgr.h" 18 #include "xe_vram_types.h" 19 20 static inline struct gpu_buddy_block * 21 xe_ttm_vram_mgr_first_block(struct list_head *list) 22 { 23 return list_first_entry_or_null(list, struct gpu_buddy_block, link); 24 } 25 26 static inline bool xe_is_vram_mgr_blocks_contiguous(struct gpu_buddy *mm, 27 struct list_head *head) 28 { 29 struct gpu_buddy_block *block; 30 u64 start, size; 31 32 block = xe_ttm_vram_mgr_first_block(head); 33 if (!block) 34 return false; 35 36 while (head != block->link.next) { 37 start = gpu_buddy_block_offset(block); 38 size = gpu_buddy_block_size(mm, block); 39 40 block = list_entry(block->link.next, struct gpu_buddy_block, 41 link); 42 if (start + size != gpu_buddy_block_offset(block)) 43 return false; 44 } 45 46 return true; 47 } 48 49 static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, 50 struct ttm_buffer_object *tbo, 51 const struct ttm_place *place, 52 struct ttm_resource **res) 53 { 54 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 55 struct xe_ttm_vram_mgr_resource *vres; 56 struct gpu_buddy *mm = &mgr->mm; 57 u64 size, min_page_size; 58 unsigned long lpfn; 59 int err; 60 61 lpfn = place->lpfn; 62 if (!lpfn || lpfn > man->size >> PAGE_SHIFT) 63 lpfn = man->size >> PAGE_SHIFT; 64 65 if (tbo->base.size >> PAGE_SHIFT > (lpfn - place->fpfn)) 66 return -E2BIG; /* don't trigger eviction for the impossible */ 67 68 vres = kzalloc_obj(*vres); 69 if (!vres) 70 return -ENOMEM; 71 72 ttm_resource_init(tbo, place, &vres->base); 73 74 /* bail out quickly if there's likely not enough VRAM for this BO */ 75 if (ttm_resource_manager_usage(man) > man->size) { 76 err = -ENOSPC; 77 goto error_fini; 78 } 79 80 INIT_LIST_HEAD(&vres->blocks); 81 82 if (place->flags & TTM_PL_FLAG_TOPDOWN) 83 vres->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION; 84 85 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) 86 vres->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION; 87 88 if (place->fpfn || lpfn != man->size >> PAGE_SHIFT) 89 vres->flags |= GPU_BUDDY_RANGE_ALLOCATION; 90 91 if (WARN_ON(!vres->base.size)) { 92 err = -EINVAL; 93 goto error_fini; 94 } 95 size = vres->base.size; 96 97 min_page_size = mgr->default_page_size; 98 if (tbo->page_alignment) 99 min_page_size = (u64)tbo->page_alignment << PAGE_SHIFT; 100 101 if (WARN_ON(min_page_size < mm->chunk_size)) { 102 err = -EINVAL; 103 goto error_fini; 104 } 105 106 if (WARN_ON(!IS_ALIGNED(size, min_page_size))) { 107 err = -EINVAL; 108 goto error_fini; 109 } 110 111 mutex_lock(&mgr->lock); 112 if (lpfn <= mgr->visible_size >> PAGE_SHIFT && size > mgr->visible_avail) { 113 err = -ENOSPC; 114 goto error_unlock; 115 } 116 117 err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, 118 (u64)lpfn << PAGE_SHIFT, size, 119 min_page_size, &vres->blocks, vres->flags); 120 if (err) 121 goto error_unlock; 122 123 if (lpfn <= mgr->visible_size >> PAGE_SHIFT) { 124 vres->used_visible_size = size; 125 } else { 126 struct gpu_buddy_block *block; 127 128 list_for_each_entry(block, &vres->blocks, link) { 129 u64 start = gpu_buddy_block_offset(block); 130 131 if (start < mgr->visible_size) { 132 u64 end = start + gpu_buddy_block_size(mm, block); 133 134 vres->used_visible_size += 135 min(end, mgr->visible_size) - start; 136 } 137 } 138 } 139 140 mgr->visible_avail -= vres->used_visible_size; 141 mutex_unlock(&mgr->lock); 142 143 if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) && 144 xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks)) 145 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; 146 147 /* 148 * For some kernel objects we still rely on the start when io mapping 149 * the object. 150 */ 151 if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) { 152 struct gpu_buddy_block *block = list_first_entry(&vres->blocks, 153 typeof(*block), 154 link); 155 156 vres->base.start = gpu_buddy_block_offset(block) >> PAGE_SHIFT; 157 } else { 158 vres->base.start = XE_BO_INVALID_OFFSET; 159 } 160 161 *res = &vres->base; 162 return 0; 163 error_unlock: 164 mutex_unlock(&mgr->lock); 165 error_fini: 166 ttm_resource_fini(man, &vres->base); 167 kfree(vres); 168 169 return err; 170 } 171 172 static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, 173 struct ttm_resource *res) 174 { 175 struct xe_ttm_vram_mgr_resource *vres = 176 to_xe_ttm_vram_mgr_resource(res); 177 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 178 struct gpu_buddy *mm = &mgr->mm; 179 180 mutex_lock(&mgr->lock); 181 gpu_buddy_free_list(mm, &vres->blocks, 0); 182 mgr->visible_avail += vres->used_visible_size; 183 mutex_unlock(&mgr->lock); 184 185 ttm_resource_fini(man, res); 186 187 kfree(vres); 188 } 189 190 static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man, 191 struct drm_printer *printer) 192 { 193 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 194 struct gpu_buddy *mm = &mgr->mm; 195 196 mutex_lock(&mgr->lock); 197 drm_printf(printer, "default_page_size: %lluKiB\n", 198 mgr->default_page_size >> 10); 199 drm_printf(printer, "visible_avail: %lluMiB\n", 200 (u64)mgr->visible_avail >> 20); 201 drm_printf(printer, "visible_size: %lluMiB\n", 202 (u64)mgr->visible_size >> 20); 203 204 drm_buddy_print(mm, printer); 205 mutex_unlock(&mgr->lock); 206 drm_printf(printer, "man size:%llu\n", man->size); 207 } 208 209 static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man, 210 struct ttm_resource *res, 211 const struct ttm_place *place, 212 size_t size) 213 { 214 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 215 struct xe_ttm_vram_mgr_resource *vres = 216 to_xe_ttm_vram_mgr_resource(res); 217 struct gpu_buddy *mm = &mgr->mm; 218 struct gpu_buddy_block *block; 219 220 if (!place->fpfn && !place->lpfn) 221 return true; 222 223 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT) 224 return vres->used_visible_size > 0; 225 226 list_for_each_entry(block, &vres->blocks, link) { 227 unsigned long fpfn = 228 gpu_buddy_block_offset(block) >> PAGE_SHIFT; 229 unsigned long lpfn = fpfn + 230 (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); 231 232 if (place->fpfn < lpfn && place->lpfn > fpfn) 233 return true; 234 } 235 236 return false; 237 } 238 239 static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man, 240 struct ttm_resource *res, 241 const struct ttm_place *place, 242 size_t size) 243 { 244 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 245 struct xe_ttm_vram_mgr_resource *vres = 246 to_xe_ttm_vram_mgr_resource(res); 247 struct gpu_buddy *mm = &mgr->mm; 248 struct gpu_buddy_block *block; 249 250 if (!place->fpfn && !place->lpfn) 251 return true; 252 253 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT) 254 return vres->used_visible_size == size; 255 256 list_for_each_entry(block, &vres->blocks, link) { 257 unsigned long fpfn = 258 gpu_buddy_block_offset(block) >> PAGE_SHIFT; 259 unsigned long lpfn = fpfn + 260 (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); 261 262 if (fpfn < place->fpfn || lpfn > place->lpfn) 263 return false; 264 } 265 266 return true; 267 } 268 269 static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = { 270 .alloc = xe_ttm_vram_mgr_new, 271 .free = xe_ttm_vram_mgr_del, 272 .intersects = xe_ttm_vram_mgr_intersects, 273 .compatible = xe_ttm_vram_mgr_compatible, 274 .debug = xe_ttm_vram_mgr_debug 275 }; 276 277 static void xe_ttm_vram_mgr_fini(struct drm_device *dev, void *arg) 278 { 279 struct xe_device *xe = to_xe_device(dev); 280 struct xe_ttm_vram_mgr *mgr = arg; 281 struct ttm_resource_manager *man = &mgr->manager; 282 283 ttm_resource_manager_set_used(man, false); 284 285 if (ttm_resource_manager_evict_all(&xe->ttm, man)) 286 return; 287 288 WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size); 289 290 gpu_buddy_fini(&mgr->mm); 291 292 ttm_resource_manager_cleanup(&mgr->manager); 293 294 ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL); 295 296 mutex_destroy(&mgr->lock); 297 } 298 299 int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, 300 u32 mem_type, u64 size, u64 io_size, 301 u64 default_page_size) 302 { 303 struct ttm_resource_manager *man = &mgr->manager; 304 int err; 305 306 if (mem_type != XE_PL_STOLEN) { 307 const char *name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; 308 man->cg = drmm_cgroup_register_region(&xe->drm, name, size); 309 if (IS_ERR(man->cg)) 310 return PTR_ERR(man->cg); 311 } 312 313 man->func = &xe_ttm_vram_mgr_func; 314 mgr->mem_type = mem_type; 315 mutex_init(&mgr->lock); 316 mgr->default_page_size = default_page_size; 317 mgr->visible_size = io_size; 318 mgr->visible_avail = io_size; 319 320 ttm_resource_manager_init(man, &xe->ttm, size); 321 err = gpu_buddy_init(&mgr->mm, man->size, default_page_size); 322 if (err) 323 return err; 324 325 ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager); 326 ttm_resource_manager_set_used(&mgr->manager, true); 327 328 return drmm_add_action_or_reset(&xe->drm, xe_ttm_vram_mgr_fini, mgr); 329 } 330 331 /** 332 * xe_ttm_vram_mgr_init - initialize TTM VRAM region 333 * @xe: pointer to Xe device 334 * @vram: pointer to xe_vram_region that contains the memory region attributes 335 * 336 * Initialize the Xe TTM for given @vram region using the given parameters. 337 * 338 * Returns 0 for success, negative error code otherwise. 339 */ 340 int xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_vram_region *vram) 341 { 342 return __xe_ttm_vram_mgr_init(xe, &vram->ttm, vram->placement, 343 xe_vram_region_usable_size(vram), 344 xe_vram_region_io_size(vram), 345 PAGE_SIZE); 346 } 347 348 int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, 349 struct ttm_resource *res, 350 u64 offset, u64 length, 351 struct device *dev, 352 enum dma_data_direction dir, 353 struct sg_table **sgt) 354 { 355 struct xe_tile *tile = &xe->tiles[res->mem_type - XE_PL_VRAM0]; 356 struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res); 357 struct xe_res_cursor cursor; 358 struct scatterlist *sg; 359 int num_entries = 0; 360 int i, r; 361 362 if (vres->used_visible_size < res->size) 363 return -EOPNOTSUPP; 364 365 *sgt = kmalloc_obj(**sgt); 366 if (!*sgt) 367 return -ENOMEM; 368 369 /* Determine the number of GPU_BUDDY blocks to export */ 370 xe_res_first(res, offset, length, &cursor); 371 while (cursor.remaining) { 372 num_entries++; 373 /* Limit maximum size to 2GiB due to SG table limitations. */ 374 xe_res_next(&cursor, min_t(u64, cursor.size, SZ_2G)); 375 } 376 377 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); 378 if (r) 379 goto error_free; 380 381 /* Initialize scatterlist nodes of sg_table */ 382 for_each_sgtable_sg((*sgt), sg, i) 383 sg->length = 0; 384 385 /* 386 * Walk down GPU_BUDDY blocks to populate scatterlist nodes 387 * @note: Use iterator api to get first the GPU_BUDDY block 388 * and the number of bytes from it. Access the following 389 * GPU_BUDDY block(s) if more buffer needs to exported 390 */ 391 xe_res_first(res, offset, length, &cursor); 392 for_each_sgtable_sg((*sgt), sg, i) { 393 phys_addr_t phys = cursor.start + xe_vram_region_io_start(tile->mem.vram); 394 size_t size = min_t(u64, cursor.size, SZ_2G); 395 dma_addr_t addr; 396 397 addr = dma_map_resource(dev, phys, size, dir, 398 DMA_ATTR_SKIP_CPU_SYNC); 399 r = dma_mapping_error(dev, addr); 400 if (r) 401 goto error_unmap; 402 403 sg_set_page(sg, NULL, size, 0); 404 sg_dma_address(sg) = addr; 405 sg_dma_len(sg) = size; 406 407 xe_res_next(&cursor, size); 408 } 409 410 return 0; 411 412 error_unmap: 413 for_each_sgtable_sg((*sgt), sg, i) { 414 if (!sg->length) 415 continue; 416 417 dma_unmap_resource(dev, sg->dma_address, 418 sg->length, dir, 419 DMA_ATTR_SKIP_CPU_SYNC); 420 } 421 sg_free_table(*sgt); 422 423 error_free: 424 kfree(*sgt); 425 return r; 426 } 427 428 void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir, 429 struct sg_table *sgt) 430 { 431 struct scatterlist *sg; 432 int i; 433 434 for_each_sgtable_sg(sgt, sg, i) 435 dma_unmap_resource(dev, sg->dma_address, 436 sg->length, dir, 437 DMA_ATTR_SKIP_CPU_SYNC); 438 sg_free_table(sgt); 439 kfree(sgt); 440 } 441 442 u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man) 443 { 444 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 445 446 return mgr->visible_size; 447 } 448 449 void xe_ttm_vram_get_used(struct ttm_resource_manager *man, 450 u64 *used, u64 *used_visible) 451 { 452 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 453 454 mutex_lock(&mgr->lock); 455 *used = mgr->mm.size - mgr->mm.avail; 456 *used_visible = mgr->visible_size - mgr->visible_avail; 457 mutex_unlock(&mgr->lock); 458 } 459 460 u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man) 461 { 462 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 463 u64 avail; 464 465 mutex_lock(&mgr->lock); 466 avail = mgr->mm.avail; 467 mutex_unlock(&mgr->lock); 468 469 return avail; 470 } 471