1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021-2022 Intel Corporation 4 * Copyright (C) 2021-2002 Red Hat 5 */ 6 7 #include <drm/drm_managed.h> 8 #include <drm/drm_drv.h> 9 10 #include <drm/ttm/ttm_placement.h> 11 #include <drm/ttm/ttm_range_manager.h> 12 13 #include "xe_bo.h" 14 #include "xe_device.h" 15 #include "xe_gt.h" 16 #include "xe_res_cursor.h" 17 #include "xe_ttm_vram_mgr.h" 18 19 static inline struct drm_buddy_block * 20 xe_ttm_vram_mgr_first_block(struct list_head *list) 21 { 22 return list_first_entry_or_null(list, struct drm_buddy_block, link); 23 } 24 25 static inline bool xe_is_vram_mgr_blocks_contiguous(struct drm_buddy *mm, 26 struct list_head *head) 27 { 28 struct drm_buddy_block *block; 29 u64 start, size; 30 31 block = xe_ttm_vram_mgr_first_block(head); 32 if (!block) 33 return false; 34 35 while (head != block->link.next) { 36 start = drm_buddy_block_offset(block); 37 size = drm_buddy_block_size(mm, block); 38 39 block = list_entry(block->link.next, struct drm_buddy_block, 40 link); 41 if (start + size != drm_buddy_block_offset(block)) 42 return false; 43 } 44 45 return true; 46 } 47 48 static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, 49 struct ttm_buffer_object *tbo, 50 const struct ttm_place *place, 51 struct ttm_resource **res) 52 { 53 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 54 struct xe_ttm_vram_mgr_resource *vres; 55 struct drm_buddy *mm = &mgr->mm; 56 u64 size, min_page_size; 57 unsigned long lpfn; 58 int err; 59 60 lpfn = place->lpfn; 61 if (!lpfn || lpfn > man->size >> PAGE_SHIFT) 62 lpfn = man->size >> PAGE_SHIFT; 63 64 if (tbo->base.size >> PAGE_SHIFT > (lpfn - place->fpfn)) 65 return -E2BIG; /* don't trigger eviction for the impossible */ 66 67 vres = kzalloc(sizeof(*vres), GFP_KERNEL); 68 if (!vres) 69 return -ENOMEM; 70 71 ttm_resource_init(tbo, place, &vres->base); 72 73 /* bail out quickly if there's likely not enough VRAM for this BO */ 74 if (ttm_resource_manager_usage(man) > man->size) { 75 err = -ENOSPC; 76 goto error_fini; 77 } 78 79 INIT_LIST_HEAD(&vres->blocks); 80 81 if (place->flags & TTM_PL_FLAG_TOPDOWN) 82 vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; 83 84 if (place->fpfn || lpfn != man->size >> PAGE_SHIFT) 85 vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; 86 87 if (WARN_ON(!vres->base.size)) { 88 err = -EINVAL; 89 goto error_fini; 90 } 91 size = vres->base.size; 92 93 min_page_size = mgr->default_page_size; 94 if (tbo->page_alignment) 95 min_page_size = (u64)tbo->page_alignment << PAGE_SHIFT; 96 97 if (WARN_ON(min_page_size < mm->chunk_size)) { 98 err = -EINVAL; 99 goto error_fini; 100 } 101 102 if (WARN_ON(!IS_ALIGNED(size, min_page_size))) { 103 err = -EINVAL; 104 goto error_fini; 105 } 106 107 mutex_lock(&mgr->lock); 108 if (lpfn <= mgr->visible_size >> PAGE_SHIFT && size > mgr->visible_avail) { 109 err = -ENOSPC; 110 goto error_unlock; 111 } 112 113 if (place->fpfn + (size >> PAGE_SHIFT) != lpfn && 114 place->flags & TTM_PL_FLAG_CONTIGUOUS) { 115 size = roundup_pow_of_two(size); 116 min_page_size = size; 117 118 lpfn = max_t(unsigned long, place->fpfn + (size >> PAGE_SHIFT), lpfn); 119 } 120 121 err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, 122 (u64)lpfn << PAGE_SHIFT, size, 123 min_page_size, &vres->blocks, vres->flags); 124 if (err) 125 goto error_unlock; 126 127 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 128 if (!drm_buddy_block_trim(mm, NULL, vres->base.size, &vres->blocks)) 129 size = vres->base.size; 130 } 131 132 if (lpfn <= mgr->visible_size >> PAGE_SHIFT) { 133 vres->used_visible_size = size; 134 } else { 135 struct drm_buddy_block *block; 136 137 list_for_each_entry(block, &vres->blocks, link) { 138 u64 start = drm_buddy_block_offset(block); 139 140 if (start < mgr->visible_size) { 141 u64 end = start + drm_buddy_block_size(mm, block); 142 143 vres->used_visible_size += 144 min(end, mgr->visible_size) - start; 145 } 146 } 147 } 148 149 mgr->visible_avail -= vres->used_visible_size; 150 mutex_unlock(&mgr->lock); 151 152 if (!(vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) && 153 xe_is_vram_mgr_blocks_contiguous(mm, &vres->blocks)) 154 vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; 155 156 /* 157 * For some kernel objects we still rely on the start when io mapping 158 * the object. 159 */ 160 if (vres->base.placement & TTM_PL_FLAG_CONTIGUOUS) { 161 struct drm_buddy_block *block = list_first_entry(&vres->blocks, 162 typeof(*block), 163 link); 164 165 vres->base.start = drm_buddy_block_offset(block) >> PAGE_SHIFT; 166 } else { 167 vres->base.start = XE_BO_INVALID_OFFSET; 168 } 169 170 *res = &vres->base; 171 return 0; 172 error_unlock: 173 mutex_unlock(&mgr->lock); 174 error_fini: 175 ttm_resource_fini(man, &vres->base); 176 kfree(vres); 177 178 return err; 179 } 180 181 static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, 182 struct ttm_resource *res) 183 { 184 struct xe_ttm_vram_mgr_resource *vres = 185 to_xe_ttm_vram_mgr_resource(res); 186 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 187 struct drm_buddy *mm = &mgr->mm; 188 189 mutex_lock(&mgr->lock); 190 drm_buddy_free_list(mm, &vres->blocks, 0); 191 mgr->visible_avail += vres->used_visible_size; 192 mutex_unlock(&mgr->lock); 193 194 ttm_resource_fini(man, res); 195 196 kfree(vres); 197 } 198 199 static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man, 200 struct drm_printer *printer) 201 { 202 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 203 struct drm_buddy *mm = &mgr->mm; 204 205 mutex_lock(&mgr->lock); 206 drm_printf(printer, "default_page_size: %lluKiB\n", 207 mgr->default_page_size >> 10); 208 drm_printf(printer, "visible_avail: %lluMiB\n", 209 (u64)mgr->visible_avail >> 20); 210 drm_printf(printer, "visible_size: %lluMiB\n", 211 (u64)mgr->visible_size >> 20); 212 213 drm_buddy_print(mm, printer); 214 mutex_unlock(&mgr->lock); 215 drm_printf(printer, "man size:%llu\n", man->size); 216 } 217 218 static bool xe_ttm_vram_mgr_intersects(struct ttm_resource_manager *man, 219 struct ttm_resource *res, 220 const struct ttm_place *place, 221 size_t size) 222 { 223 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 224 struct xe_ttm_vram_mgr_resource *vres = 225 to_xe_ttm_vram_mgr_resource(res); 226 struct drm_buddy *mm = &mgr->mm; 227 struct drm_buddy_block *block; 228 229 if (!place->fpfn && !place->lpfn) 230 return true; 231 232 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT) 233 return vres->used_visible_size > 0; 234 235 list_for_each_entry(block, &vres->blocks, link) { 236 unsigned long fpfn = 237 drm_buddy_block_offset(block) >> PAGE_SHIFT; 238 unsigned long lpfn = fpfn + 239 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); 240 241 if (place->fpfn < lpfn && place->lpfn > fpfn) 242 return true; 243 } 244 245 return false; 246 } 247 248 static bool xe_ttm_vram_mgr_compatible(struct ttm_resource_manager *man, 249 struct ttm_resource *res, 250 const struct ttm_place *place, 251 size_t size) 252 { 253 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 254 struct xe_ttm_vram_mgr_resource *vres = 255 to_xe_ttm_vram_mgr_resource(res); 256 struct drm_buddy *mm = &mgr->mm; 257 struct drm_buddy_block *block; 258 259 if (!place->fpfn && !place->lpfn) 260 return true; 261 262 if (!place->fpfn && place->lpfn == mgr->visible_size >> PAGE_SHIFT) 263 return vres->used_visible_size == size; 264 265 list_for_each_entry(block, &vres->blocks, link) { 266 unsigned long fpfn = 267 drm_buddy_block_offset(block) >> PAGE_SHIFT; 268 unsigned long lpfn = fpfn + 269 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); 270 271 if (fpfn < place->fpfn || lpfn > place->lpfn) 272 return false; 273 } 274 275 return true; 276 } 277 278 static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = { 279 .alloc = xe_ttm_vram_mgr_new, 280 .free = xe_ttm_vram_mgr_del, 281 .intersects = xe_ttm_vram_mgr_intersects, 282 .compatible = xe_ttm_vram_mgr_compatible, 283 .debug = xe_ttm_vram_mgr_debug 284 }; 285 286 static void ttm_vram_mgr_fini(struct drm_device *dev, void *arg) 287 { 288 struct xe_device *xe = to_xe_device(dev); 289 struct xe_ttm_vram_mgr *mgr = arg; 290 struct ttm_resource_manager *man = &mgr->manager; 291 292 ttm_resource_manager_set_used(man, false); 293 294 if (ttm_resource_manager_evict_all(&xe->ttm, man)) 295 return; 296 297 WARN_ON_ONCE(mgr->visible_avail != mgr->visible_size); 298 299 drm_buddy_fini(&mgr->mm); 300 301 ttm_resource_manager_cleanup(&mgr->manager); 302 303 ttm_set_driver_manager(&xe->ttm, mgr->mem_type, NULL); 304 305 mutex_destroy(&mgr->lock); 306 } 307 308 int __xe_ttm_vram_mgr_init(struct xe_device *xe, struct xe_ttm_vram_mgr *mgr, 309 u32 mem_type, u64 size, u64 io_size, 310 u64 default_page_size) 311 { 312 struct ttm_resource_manager *man = &mgr->manager; 313 int err; 314 315 if (mem_type != XE_PL_STOLEN) { 316 const char *name = mem_type == XE_PL_VRAM0 ? "vram0" : "vram1"; 317 man->cg = drmm_cgroup_register_region(&xe->drm, name, size); 318 if (IS_ERR(man->cg)) 319 return PTR_ERR(man->cg); 320 } 321 322 man->func = &xe_ttm_vram_mgr_func; 323 mgr->mem_type = mem_type; 324 mutex_init(&mgr->lock); 325 mgr->default_page_size = default_page_size; 326 mgr->visible_size = io_size; 327 mgr->visible_avail = io_size; 328 329 ttm_resource_manager_init(man, &xe->ttm, size); 330 err = drm_buddy_init(&mgr->mm, man->size, default_page_size); 331 if (err) 332 return err; 333 334 ttm_set_driver_manager(&xe->ttm, mem_type, &mgr->manager); 335 ttm_resource_manager_set_used(&mgr->manager, true); 336 337 return drmm_add_action_or_reset(&xe->drm, ttm_vram_mgr_fini, mgr); 338 } 339 340 int xe_ttm_vram_mgr_init(struct xe_tile *tile, struct xe_ttm_vram_mgr *mgr) 341 { 342 struct xe_device *xe = tile_to_xe(tile); 343 struct xe_mem_region *vram = &tile->mem.vram; 344 345 mgr->vram = vram; 346 return __xe_ttm_vram_mgr_init(xe, mgr, XE_PL_VRAM0 + tile->id, 347 vram->usable_size, vram->io_size, 348 PAGE_SIZE); 349 } 350 351 int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, 352 struct ttm_resource *res, 353 u64 offset, u64 length, 354 struct device *dev, 355 enum dma_data_direction dir, 356 struct sg_table **sgt) 357 { 358 struct xe_tile *tile = &xe->tiles[res->mem_type - XE_PL_VRAM0]; 359 struct xe_ttm_vram_mgr_resource *vres = to_xe_ttm_vram_mgr_resource(res); 360 struct xe_res_cursor cursor; 361 struct scatterlist *sg; 362 int num_entries = 0; 363 int i, r; 364 365 if (vres->used_visible_size < res->size) 366 return -EOPNOTSUPP; 367 368 *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL); 369 if (!*sgt) 370 return -ENOMEM; 371 372 /* Determine the number of DRM_BUDDY blocks to export */ 373 xe_res_first(res, offset, length, &cursor); 374 while (cursor.remaining) { 375 num_entries++; 376 /* Limit maximum size to 2GiB due to SG table limitations. */ 377 xe_res_next(&cursor, min_t(u64, cursor.size, SZ_2G)); 378 } 379 380 r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); 381 if (r) 382 goto error_free; 383 384 /* Initialize scatterlist nodes of sg_table */ 385 for_each_sgtable_sg((*sgt), sg, i) 386 sg->length = 0; 387 388 /* 389 * Walk down DRM_BUDDY blocks to populate scatterlist nodes 390 * @note: Use iterator api to get first the DRM_BUDDY block 391 * and the number of bytes from it. Access the following 392 * DRM_BUDDY block(s) if more buffer needs to exported 393 */ 394 xe_res_first(res, offset, length, &cursor); 395 for_each_sgtable_sg((*sgt), sg, i) { 396 phys_addr_t phys = cursor.start + tile->mem.vram.io_start; 397 size_t size = min_t(u64, cursor.size, SZ_2G); 398 dma_addr_t addr; 399 400 addr = dma_map_resource(dev, phys, size, dir, 401 DMA_ATTR_SKIP_CPU_SYNC); 402 r = dma_mapping_error(dev, addr); 403 if (r) 404 goto error_unmap; 405 406 sg_set_page(sg, NULL, size, 0); 407 sg_dma_address(sg) = addr; 408 sg_dma_len(sg) = size; 409 410 xe_res_next(&cursor, size); 411 } 412 413 return 0; 414 415 error_unmap: 416 for_each_sgtable_sg((*sgt), sg, i) { 417 if (!sg->length) 418 continue; 419 420 dma_unmap_resource(dev, sg->dma_address, 421 sg->length, dir, 422 DMA_ATTR_SKIP_CPU_SYNC); 423 } 424 sg_free_table(*sgt); 425 426 error_free: 427 kfree(*sgt); 428 return r; 429 } 430 431 void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir, 432 struct sg_table *sgt) 433 { 434 struct scatterlist *sg; 435 int i; 436 437 for_each_sgtable_sg(sgt, sg, i) 438 dma_unmap_resource(dev, sg->dma_address, 439 sg->length, dir, 440 DMA_ATTR_SKIP_CPU_SYNC); 441 sg_free_table(sgt); 442 kfree(sgt); 443 } 444 445 u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man) 446 { 447 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 448 449 return mgr->visible_size; 450 } 451 452 void xe_ttm_vram_get_used(struct ttm_resource_manager *man, 453 u64 *used, u64 *used_visible) 454 { 455 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 456 457 mutex_lock(&mgr->lock); 458 *used = mgr->mm.size - mgr->mm.avail; 459 *used_visible = mgr->visible_size - mgr->visible_avail; 460 mutex_unlock(&mgr->lock); 461 } 462 463 u64 xe_ttm_vram_get_avail(struct ttm_resource_manager *man) 464 { 465 struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); 466 u64 avail; 467 468 mutex_lock(&mgr->lock); 469 avail = mgr->mm.avail; 470 mutex_unlock(&mgr->lock); 471 472 return avail; 473 } 474