1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021-2022 Intel Corporation 4 * Copyright (C) 2021-2002 Red Hat 5 */ 6 7 #include <drm/drm_managed.h> 8 #include <drm/drm_drv.h> 9 10 #include <drm/ttm/ttm_placement.h> 11 #include <drm/ttm/ttm_range_manager.h> 12 13 #include "xe_bo.h" 14 #include "xe_device.h" 15 #include "xe_gt.h" 16 #include "xe_res_cursor.h" 17 #include "xe_ttm_vram_mgr.h" 18 #include "xe_vram_types.h" 19 20 static inline struct drm_buddy_block * 21 xe_ttm_vram_mgr_first_block(struct list_head *list) 22 { 23 return list_first_entry_or_null(list, struct drm_buddy_block, link); 24 } 25 26 static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm, 27 struct list_head *head) 28 { 29 struct drm_buddy_block *block; 30 u64 start, size; 31 32 block = xe_ttm_vram_mgr_first_block(head); 33 if (!block) 34 return false; 35 36 while (head != block->link.next) { 37 start = drm_buddy_block_offset(block); 38 size = drm_buddy_block_size(mm, block); 39 40 block = list_entry(block->link.next, struct drm_buddy_block, 41 link); 42 if (start + size != drm_buddy_block_offset(block)) 43 return false; 44 } 45 46 return true; 47 } 48 49 static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, 50 struct ttm_buffer_object *tbo, 51 const struct ttm_place *place, 52 struct ttm_resource **res) 53 { 54 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 55 struct xe_ttm_vram_mgr_resource *vres; 56 struct drm_buddy *mm = &mgr->mm; 57 u64 size, min_page_size; 58 unsigned long lpfn; 59 int err; 60 61 lpfn = place->lpfn; 62 if (!lpfn || lpfn > man->size >> PAGE_SHIFT) 63 lpfn = man->size >> PAGE_SHIFT; 64 65 if (tbo->base.size >> PAGE_SHIFT > (lpfn - place->fpfn)) 66 return -E2BIG; /* don't trigger eviction for the impossible */ 67 68 vres = kzalloc(sizeof(*vres), GFP_KERNEL); 69 if (!vres) 70 return -ENOMEM; 71 72 ttm_resource_init(tbo, place, &vres->base); 73 74 /* bail out quickly if there's likely not enough VRAM for this BO */ 75 if (ttm_resource_manager_usage(man) > man->size) { 76 err = -ENOSPC; 77 goto error_fini; 78 } 79 80 INIT_LIST_HEAD(&vres->blocks); 81 82 if (place->flags & TTM_PL_FLAG_TOPDOWN) 83 vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; 84 85 if (place->fpfn || lpfn != man->size >> PAGE_SHIFT) 86 vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; 87 88 if (WARN_ON(!vres->base.size)) { 89 err = -EINVAL; 90 goto error_fini; 91 } 92 size = vres->base.size; 93 94 min_page_size = mgr->default_page_size; 95 if (tbo->page_alignment) 96 min_page_size = (u64)tbo->page_alignment << PAGE_SHIFT; 97 98 if (WARN_ON(min_page_size < mm->chunk_size)) { 99 err = -EINVAL; 100 goto error_fini; 101 } 102 103 if (WARN_ON(!IS_ALIGNED(size, min_page_size))) { 104 err = -EINVAL; 105 goto error_fini; 106 } 107 108 mutex_lock(&mgr->lock); 109 if (lpfn <= mgr->visible_size >> PAGE_SHIFT && size > mgr->visible_avail) { 110 err = -ENOSPC; 111 goto error_unlock; 112 } 113 114 if (place->fpfn + (size >> PAGE_SHIFT) != lpfn && 115 place->flags & TTM_PL_FLAG_CONTIGUOUS) { 116 size = roundup_pow_of_two(size); 117 min_page_size = size; 118 119 lpfn = max_t(unsigned long, place->fpfn + (size >> PAGE_SHIFT), lpfn); 120 } 121 122 err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, 123 (u64)lpfn << PAGE_SHIFT, size, 124 min_page_size, &vres->blocks, vres->flags); 125 if (err) 126 goto error_unlock; 127 128 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 129 if (!drm_buddy_block_trim(mm, NULL, vres->base.size, &vres->blocks)) 130 size = vres->base.size; 131 } 132 133 if (lpfn <= mgr->visible_size >> PAGE_SHIFT) { 134 vres->used_visible_size = size; 135 } else { 136 struct drm_buddy_block *block; 137 138 list_for_each_entry(block, &vres->blocks, link) { 139 u64 start = drm_buddy_block_offset(block); 140 141 if (start < mgr->visible_size) { 142 u64 end = start + drm_buddy_block_size(mm, block); 143 144 vres->used_visible_size += 145 min(end, mgr->visible_size) - start; 146 } 147 } 148 } 149 150 mgr->visible_avail -= vres->used_visible_size; 151 mutex_unlock(&mgr->lock); 152 153 if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) && 154 xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks)) 155 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; 156 157 /* 158 * For some kernel objects we still rely on the start when io mapping 159 * the object. 160 */ 161 if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) { 162 struct drm_buddy_block *block = list_first_entry(&vres->blocks, 163 typeof(*block), 164 link); 165 166 vres->base.start = drm_buddy_block_offset(block) >> PAGE_SHIFT; 167 } else { 168 vres->base.start = XE_BO_INVALID_OFFSET; 169 } 170 171 *res = &vres->base; 172 return 0; 173 error_unlock: 174 mutex_unlock(&mgr->lock); 175 error_fini: 176 ttm_resource_fini(man, &vres->base); 177 kfree(vres); 178 179 return err; 180 } 181 182 static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, 183 struct ttm_resource *res) 184 { 185 struct xe_ttm_vram_mgr_resource *vres = 186 to_xe_ttm_vram_mgr_resource(res); 187 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 188 struct drm_buddy *mm = &mgr->mm; 189 190 mutex_lock(&mgr->lock); 191 drm_buddy_free_list(mm, &vres->blocks, 0); 192 mgr->visible_avail += vres->used_visible_size; 193 mutex_unlock(&mgr->lock); 194 195 ttm_resource_fini(man, res); 196 197 kfree(vres); 198 } 199 200 static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man, 201 struct drm_printer *printer) 202 { 203 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 204 struct drm_buddy *mm = &mgr->mm; 205 206 mutex_lock(&mgr->lock); 207 drm_printf(printer, "default_page_size: %lluKiB\n", 208 mgr->default_page_size >> 10); 209 drm_printf(printer, "visible_avail: %lluMiB\n", 210 (u64)mgr->visible_avail >> 20); 211 drm_printf(printer, "visible_size: %lluMiB\n", 212 (u64)mgr->visible_size >> 20); 213 214 drm_buddy_print(mm, printer); 215 mutex_unlock(&mgr->lock); 216 drm_printf(printer, "man size:%llu\n", man->size); 217 } 218 219 static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man, 220 struct ttm_resource *res, 221 const struct ttm_place *place, 222 size_t size) 223 { 224 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 225 struct xe_ttm_vram_mgr_resource *vres = 226 to_xe_ttm_vram_mgr_resource(res); 227 struct drm_buddy *mm = &mgr->mm; 228 struct drm_buddy_block *block; 229 230 if (!place->fpfn && !place->lpfn) 231 return true; 232 233 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT) 234 return vres->used_visible_size > 0; 235 236 list_for_each_entry(block, &vres->blocks, link) { 237 unsigned long fpfn = 238 drm_buddy_block_offset(block) >> PAGE_SHIFT; 239 unsigned long lpfn = fpfn + 240 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); 241 242 if (place->fpfn < lpfn && place->lpfn > fpfn) 243 return true; 244 } 245 246 return false; 247 } 248 249 static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man, 250 struct ttm_resource *res, 251 const struct ttm_place *place, 252 size_t size) 253 { 254 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 255 struct xe_ttm_vram_mgr_resource *vres = 256 to_xe_ttm_vram_mgr_resource(res); 257 struct drm_buddy *mm = &mgr->mm; 258 struct drm_buddy_block *block; 259 260 if (!place->fpfn && !place->lpfn) 261 return true; 262 263 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT) 264 return vres->used_visible_size == size; 265 266 list_for_each_entry(block, &vres->blocks, link) { 267 unsigned long fpfn = 268 drm_buddy_block_offset(block) >> PAGE_SHIFT; 269 unsigned long lpfn = fpfn + 270 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); 271 272 if (fpfn < place->fpfn || lpfn > place->lpfn) 273 return false; 274 } 275 276 return true; 277 } 278 279 static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = { 280 .alloc = xe_ttm_vram_mgr_new, 281 .free = xe_ttm_vram_mgr_del, 282 .intersects = xe_ttm_vram_mgr_intersects, 283 .compatible = xe_ttm_vram_mgr_compatible, 284 .debug = xe_ttm_vram_mgr_debug 285 }; 286 287 static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg) 288 { 289 struct xe_device *xe = to_xe_device(dev); 290 struct xe_ttm_vram_mgr *mgr = arg; 291 struct ttm_resource_manager *man = &mgr->manager; 292 293 ttm_resource_manager_set_used(man, false); 294 295 if (ttm_resource_manager_evict_all(&xe->ttm, man)) 296 return; 297 298 WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size); 299 300 drm_buddy_fini(&mgr->mm); 301 302 ttm_resource_manager_cleanup(&mgr->manager); 303 304 ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL); 305 306 mutex_destroy(&mgr->lock); 307 } 308 309 int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, 310 u32 mem_type, u64 size, u64 io_size, 311 u64 default_page_size) 312 { 313 struct ttm_resource_manager *man = &mgr->manager; 314 int err; 315 316 if (mem_type != XE_PL_STOLEN) { 317 const char *name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; 318 man->cg = drmm_cgroup_register_region(&xe->drm, name, size); 319 if (IS_ERR(man->cg)) 320 return PTR_ERR(man->cg); 321 } 322 323 man->func = &xe_ttm_vram_mgr_func; 324 mgr->mem_type = mem_type; 325 mutex_init(&mgr->lock); 326 mgr->default_page_size = default_page_size; 327 mgr->visible_size = io_size; 328 mgr->visible_avail = io_size; 329 330 ttm_resource_manager_init(man, &xe->ttm, size); 331 err = drm_buddy_init(&mgr->mm, man->size, default_page_size); 332 if (err) 333 return err; 334 335 ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager); 336 ttm_resource_manager_set_used(&mgr->manager, true); 337 338 return drmm_add_action_or_reset(&xe->drm, ttm_vram_mgr_fini, mgr); 339 } 340 341 /** 342 * xe_ttm_vram_mgr_init - initialize TTM VRAM region 343 * @xe: pointer to Xe device 344 * @vram: pointer to xe_vram_region that contains the memory region attributes 345 * 346 * Initialize the Xe TTM for given @vram region using the given parameters. 347 * 348 * Returns 0 for success, negative error code otherwise. 349 */ 350 int xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_vram_region *vram) 351 { 352 return __xe_ttm_vram_mgr_init(xe, &vram->ttm, vram->placement, 353 xe_vram_region_usable_size(vram), 354 xe_vram_region_io_size(vram), 355 PAGE_SIZE); 356 } 357 358 int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, 359 struct ttm_resource *res, 360 u64 offset, u64 length, 361 struct device *dev, 362 enum dma_data_direction dir, 363 struct sg_table **sgt) 364 { 365 struct xe_tile *tile = &xe->tiles[res->mem_type - XE_PL_VRAM0]; 366 struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res); 367 struct xe_res_cursor cursor; 368 struct scatterlist *sg; 369 int num_entries = 0; 370 int i, r; 371 372 if (vres->used_visible_size < res->size) 373 return -EOPNOTSUPP; 374 375 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL); 376 if (!*sgt) 377 return -ENOMEM; 378 379 /* Determine the number of DRM_BUDDY blocks to export */ 380 xe_res_first(res, offset, length, &cursor); 381 while (cursor.remaining) { 382 num_entries++; 383 /* Limit maximum size to 2GiB due to SG table limitations. */ 384 xe_res_next(&cursor, min_t(u64, cursor.size, SZ_2G)); 385 } 386 387 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); 388 if (r) 389 goto error_free; 390 391 /* Initialize scatterlist nodes of sg_table */ 392 for_each_sgtable_sg((*sgt), sg, i) 393 sg->length = 0; 394 395 /* 396 * Walk down DRM_BUDDY blocks to populate scatterlist nodes 397 * @note: Use iterator api to get first the DRM_BUDDY block 398 * and the number of bytes from it. Access the following 399 * DRM_BUDDY block(s) if more buffer needs to exported 400 */ 401 xe_res_first(res, offset, length, &cursor); 402 for_each_sgtable_sg((*sgt), sg, i) { 403 phys_addr_t phys = cursor.start + xe_vram_region_io_start(tile->mem.vram); 404 size_t size = min_t(u64, cursor.size, SZ_2G); 405 dma_addr_t addr; 406 407 addr = dma_map_resource(dev, phys, size, dir, 408 DMA_ATTR_SKIP_CPU_SYNC); 409 r = dma_mapping_error(dev, addr); 410 if (r) 411 goto error_unmap; 412 413 sg_set_page(sg, NULL, size, 0); 414 sg_dma_address(sg) = addr; 415 sg_dma_len(sg) = size; 416 417 xe_res_next(&cursor, size); 418 } 419 420 return 0; 421 422 error_unmap: 423 for_each_sgtable_sg((*sgt), sg, i) { 424 if (!sg->length) 425 continue; 426 427 dma_unmap_resource(dev, sg->dma_address, 428 sg->length, dir, 429 DMA_ATTR_SKIP_CPU_SYNC); 430 } 431 sg_free_table(*sgt); 432 433 error_free: 434 kfree(*sgt); 435 return r; 436 } 437 438 void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir, 439 struct sg_table *sgt) 440 { 441 struct scatterlist *sg; 442 int i; 443 444 for_each_sgtable_sg(sgt, sg, i) 445 dma_unmap_resource(dev, sg->dma_address, 446 sg->length, dir, 447 DMA_ATTR_SKIP_CPU_SYNC); 448 sg_free_table(sgt); 449 kfree(sgt); 450 } 451 452 u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man) 453 { 454 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 455 456 return mgr->visible_size; 457 } 458 459 void xe_ttm_vram_get_used(struct ttm_resource_manager *man, 460 u64 *used, u64 *used_visible) 461 { 462 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 463 464 mutex_lock(&mgr->lock); 465 *used = mgr->mm.size - mgr->mm.avail; 466 *used_visible = mgr->visible_size - mgr->visible_avail; 467 mutex_unlock(&mgr->lock); 468 } 469 470 u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man) 471 { 472 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 473 u64 avail; 474 475 mutex_lock(&mgr->lock); 476 avail = mgr->mm.avail; 477 mutex_unlock(&mgr->lock); 478 479 return avail; 480 } 481