1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <linux/slab.h> 7 8 #include <linux/gpu_buddy.h> 9 #include <drm/drm_buddy.h> 10 #include <drm/drm_print.h> 11 #include <drm/ttm/ttm_placement.h> 12 #include <drm/ttm/ttm_bo.h> 13 14 #include "i915_ttm_buddy_manager.h" 15 16 #include "i915_gem.h" 17 18 struct i915_ttm_buddy_manager { 19 struct ttm_resource_manager manager; 20 struct gpu_buddy mm; 21 struct list_head reserved; 22 struct mutex lock; 23 unsigned long visible_size; 24 unsigned long visible_avail; 25 unsigned long visible_reserved; 26 u64 default_page_size; 27 }; 28 29 static struct i915_ttm_buddy_manager * 30 to_buddy_manager(struct ttm_resource_manager *man) 31 { 32 return container_of(man, struct i915_ttm_buddy_manager, manager); 33 } 34 35 static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, 36 struct ttm_buffer_object *bo, 37 const struct ttm_place *place, 38 struct ttm_resource **res) 39 { 40 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 41 struct i915_ttm_buddy_resource *bman_res; 42 struct gpu_buddy *mm = &bman->mm; 43 unsigned long n_pages, lpfn; 44 u64 min_page_size; 45 u64 size; 46 int err; 47 48 lpfn = place->lpfn; 49 if (!lpfn) 50 lpfn = man->size; 51 52 bman_res = kzalloc_obj(*bman_res); 53 if (!bman_res) 54 return -ENOMEM; 55 56 ttm_resource_init(bo, place, &bman_res->base); 57 INIT_LIST_HEAD(&bman_res->blocks); 58 bman_res->mm = mm; 59 60 if (place->flags & TTM_PL_FLAG_TOPDOWN) 61 bman_res->flags |= GPU_BUDDY_TOPDOWN_ALLOCATION; 62 63 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) 64 bman_res->flags |= GPU_BUDDY_CONTIGUOUS_ALLOCATION; 65 66 if (place->fpfn || lpfn != man->size) 67 bman_res->flags |= GPU_BUDDY_RANGE_ALLOCATION; 68 69 GEM_BUG_ON(!bman_res->base.size); 70 size = bman_res->base.size; 71 72 min_page_size = bman->default_page_size; 73 if (bo->page_alignment) 74 min_page_size = bo->page_alignment << PAGE_SHIFT; 75 76 GEM_BUG_ON(min_page_size < mm->chunk_size); 77 GEM_BUG_ON(!IS_ALIGNED(size, min_page_size)); 78 79 if (size > lpfn << PAGE_SHIFT) { 80 err = -E2BIG; 81 goto err_free_res; 82 } 83 84 n_pages = size >> ilog2(mm->chunk_size); 85 86 mutex_lock(&bman->lock); 87 if (lpfn <= bman->visible_size && n_pages > bman->visible_avail) { 88 mutex_unlock(&bman->lock); 89 err = -ENOSPC; 90 goto err_free_res; 91 } 92 93 err = gpu_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, 94 (u64)lpfn << PAGE_SHIFT, 95 (u64)n_pages << PAGE_SHIFT, 96 min_page_size, 97 &bman_res->blocks, 98 bman_res->flags); 99 if (unlikely(err)) 100 goto err_free_blocks; 101 102 if (lpfn <= bman->visible_size) { 103 bman_res->used_visible_size = PFN_UP(bman_res->base.size); 104 } else { 105 struct gpu_buddy_block *block; 106 107 list_for_each_entry(block, &bman_res->blocks, link) { 108 unsigned long start = 109 gpu_buddy_block_offset(block) >> PAGE_SHIFT; 110 111 if (start < bman->visible_size) { 112 unsigned long end = start + 113 (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); 114 115 bman_res->used_visible_size += 116 min(end, bman->visible_size) - start; 117 } 118 } 119 } 120 121 if (bman_res->used_visible_size) 122 bman->visible_avail -= bman_res->used_visible_size; 123 124 mutex_unlock(&bman->lock); 125 126 *res = &bman_res->base; 127 return 0; 128 129 err_free_blocks: 130 gpu_buddy_free_list(mm, &bman_res->blocks, 0); 131 mutex_unlock(&bman->lock); 132 err_free_res: 133 ttm_resource_fini(man, &bman_res->base); 134 kfree(bman_res); 135 return err; 136 } 137 138 static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man, 139 struct ttm_resource *res) 140 { 141 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); 142 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 143 144 mutex_lock(&bman->lock); 145 gpu_buddy_free_list(&bman->mm, &bman_res->blocks, 0); 146 bman->visible_avail += bman_res->used_visible_size; 147 mutex_unlock(&bman->lock); 148 149 ttm_resource_fini(man, res); 150 kfree(bman_res); 151 } 152 153 static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man, 154 struct ttm_resource *res, 155 const struct ttm_place *place, 156 size_t size) 157 { 158 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); 159 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 160 struct gpu_buddy *mm = &bman->mm; 161 struct gpu_buddy_block *block; 162 163 if (!place->fpfn && !place->lpfn) 164 return true; 165 166 GEM_BUG_ON(!place->lpfn); 167 168 /* 169 * If we just want something mappable then we can quickly check 170 * if the current victim resource is using any of the CPU 171 * visible portion. 172 */ 173 if (!place->fpfn && 174 place->lpfn == i915_ttm_buddy_man_visible_size(man)) 175 return bman_res->used_visible_size > 0; 176 177 /* Check each drm buddy block individually */ 178 list_for_each_entry(block, &bman_res->blocks, link) { 179 unsigned long fpfn = 180 gpu_buddy_block_offset(block) >> PAGE_SHIFT; 181 unsigned long lpfn = fpfn + 182 (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); 183 184 if (place->fpfn < lpfn && place->lpfn > fpfn) 185 return true; 186 } 187 188 return false; 189 } 190 191 static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man, 192 struct ttm_resource *res, 193 const struct ttm_place *place, 194 size_t size) 195 { 196 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); 197 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 198 struct gpu_buddy *mm = &bman->mm; 199 struct gpu_buddy_block *block; 200 201 if (!place->fpfn && !place->lpfn) 202 return true; 203 204 GEM_BUG_ON(!place->lpfn); 205 206 if (!place->fpfn && 207 place->lpfn == i915_ttm_buddy_man_visible_size(man)) 208 return bman_res->used_visible_size == PFN_UP(res->size); 209 210 /* Check each drm buddy block individually */ 211 list_for_each_entry(block, &bman_res->blocks, link) { 212 unsigned long fpfn = 213 gpu_buddy_block_offset(block) >> PAGE_SHIFT; 214 unsigned long lpfn = fpfn + 215 (gpu_buddy_block_size(mm, block) >> PAGE_SHIFT); 216 217 if (fpfn < place->fpfn || lpfn > place->lpfn) 218 return false; 219 } 220 221 return true; 222 } 223 224 static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man, 225 struct drm_printer *printer) 226 { 227 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 228 struct gpu_buddy_block *block; 229 230 mutex_lock(&bman->lock); 231 drm_printf(printer, "default_page_size: %lluKiB\n", 232 bman->default_page_size >> 10); 233 drm_printf(printer, "visible_avail: %lluMiB\n", 234 (u64)bman->visible_avail << PAGE_SHIFT >> 20); 235 drm_printf(printer, "visible_size: %lluMiB\n", 236 (u64)bman->visible_size << PAGE_SHIFT >> 20); 237 drm_printf(printer, "visible_reserved: %lluMiB\n", 238 (u64)bman->visible_reserved << PAGE_SHIFT >> 20); 239 240 drm_buddy_print(&bman->mm, printer); 241 242 drm_printf(printer, "reserved:\n"); 243 list_for_each_entry(block, &bman->reserved, link) 244 drm_buddy_block_print(&bman->mm, block, printer); 245 mutex_unlock(&bman->lock); 246 } 247 248 static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = { 249 .alloc = i915_ttm_buddy_man_alloc, 250 .free = i915_ttm_buddy_man_free, 251 .intersects = i915_ttm_buddy_man_intersects, 252 .compatible = i915_ttm_buddy_man_compatible, 253 .debug = i915_ttm_buddy_man_debug, 254 }; 255 256 /** 257 * i915_ttm_buddy_man_init - Setup buddy allocator based ttm manager 258 * @bdev: The ttm device 259 * @type: Memory type we want to manage 260 * @use_tt: Set use_tt for the manager 261 * @size: The size in bytes to manage 262 * @visible_size: The CPU visible size in bytes to manage 263 * @default_page_size: The default minimum page size in bytes for allocations, 264 * this must be at least as large as @chunk_size, and can be overridden by 265 * setting the BO page_alignment, to be larger or smaller as needed. 266 * @chunk_size: The minimum page size in bytes for our allocations i.e 267 * order-zero 268 * 269 * Note that the starting address is assumed to be zero here, since this 270 * simplifies keeping the property where allocated blocks having natural 271 * power-of-two alignment. So long as the real starting address is some large 272 * power-of-two, or naturally start from zero, then this should be fine. Also 273 * the &i915_ttm_buddy_man_reserve interface can be used to preserve alignment 274 * if say there is some unusable range from the start of the region. We can 275 * revisit this in the future and make the interface accept an actual starting 276 * offset and let it take care of the rest. 277 * 278 * Note that if the @size is not aligned to the @chunk_size then we perform the 279 * required rounding to get the usable size. The final size in pages can be 280 * taken from &ttm_resource_manager.size. 281 * 282 * Return: 0 on success, negative error code on failure. 283 */ 284 int i915_ttm_buddy_man_init(struct ttm_device *bdev, 285 unsigned int type, bool use_tt, 286 u64 size, u64 visible_size, u64 default_page_size, 287 u64 chunk_size) 288 { 289 struct ttm_resource_manager *man; 290 struct i915_ttm_buddy_manager *bman; 291 int err; 292 293 bman = kzalloc_obj(*bman); 294 if (!bman) 295 return -ENOMEM; 296 297 err = gpu_buddy_init(&bman->mm, size, chunk_size); 298 if (err) 299 goto err_free_bman; 300 301 mutex_init(&bman->lock); 302 INIT_LIST_HEAD(&bman->reserved); 303 GEM_BUG_ON(default_page_size < chunk_size); 304 bman->default_page_size = default_page_size; 305 bman->visible_size = visible_size >> PAGE_SHIFT; 306 bman->visible_avail = bman->visible_size; 307 308 man = &bman->manager; 309 man->use_tt = use_tt; 310 man->func = &i915_ttm_buddy_manager_func; 311 ttm_resource_manager_init(man, bdev, bman->mm.size >> PAGE_SHIFT); 312 313 ttm_resource_manager_set_used(man, true); 314 ttm_set_driver_manager(bdev, type, man); 315 316 return 0; 317 318 err_free_bman: 319 kfree(bman); 320 return err; 321 } 322 323 /** 324 * i915_ttm_buddy_man_fini - Destroy the buddy allocator ttm manager 325 * @bdev: The ttm device 326 * @type: Memory type we want to manage 327 * 328 * Note that if we reserved anything with &i915_ttm_buddy_man_reserve, this will 329 * also be freed for us here. 330 * 331 * Return: 0 on success, negative error code on failure. 332 */ 333 int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type) 334 { 335 struct ttm_resource_manager *man = ttm_manager_type(bdev, type); 336 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 337 struct gpu_buddy *mm = &bman->mm; 338 int ret; 339 340 ttm_resource_manager_set_used(man, false); 341 342 ret = ttm_resource_manager_evict_all(bdev, man); 343 if (ret) 344 return ret; 345 346 ttm_set_driver_manager(bdev, type, NULL); 347 348 mutex_lock(&bman->lock); 349 gpu_buddy_free_list(mm, &bman->reserved, 0); 350 gpu_buddy_fini(mm); 351 bman->visible_avail += bman->visible_reserved; 352 WARN_ON_ONCE(bman->visible_avail != bman->visible_size); 353 mutex_unlock(&bman->lock); 354 355 ttm_resource_manager_cleanup(man); 356 kfree(bman); 357 358 return 0; 359 } 360 361 /** 362 * i915_ttm_buddy_man_reserve - Reserve address range 363 * @man: The buddy allocator ttm manager 364 * @start: The offset in bytes, where the region start is assumed to be zero 365 * @size: The size in bytes 366 * 367 * Note that the starting address for the region is always assumed to be zero. 368 * 369 * Return: 0 on success, negative error code on failure. 370 */ 371 int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, 372 u64 start, u64 size) 373 { 374 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 375 struct gpu_buddy *mm = &bman->mm; 376 unsigned long fpfn = start >> PAGE_SHIFT; 377 unsigned long flags = 0; 378 int ret; 379 380 flags |= GPU_BUDDY_RANGE_ALLOCATION; 381 382 mutex_lock(&bman->lock); 383 ret = gpu_buddy_alloc_blocks(mm, start, 384 start + size, 385 size, mm->chunk_size, 386 &bman->reserved, 387 flags); 388 389 if (fpfn < bman->visible_size) { 390 unsigned long lpfn = fpfn + (size >> PAGE_SHIFT); 391 unsigned long visible = min(lpfn, bman->visible_size) - fpfn; 392 393 bman->visible_reserved += visible; 394 bman->visible_avail -= visible; 395 } 396 mutex_unlock(&bman->lock); 397 398 return ret; 399 } 400 401 /** 402 * i915_ttm_buddy_man_visible_size - Return the size of the CPU visible portion 403 * in pages. 404 * @man: The buddy allocator ttm manager 405 */ 406 u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man) 407 { 408 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 409 410 return bman->visible_size; 411 } 412 413 /** 414 * i915_ttm_buddy_man_avail - Query the avail tracking for the manager. 415 * 416 * @man: The buddy allocator ttm manager 417 * @avail: The total available memory in pages for the entire manager. 418 * @visible_avail: The total available memory in pages for the CPU visible 419 * portion. Note that this will always give the same value as @avail on 420 * configurations that don't have a small BAR. 421 */ 422 void i915_ttm_buddy_man_avail(struct ttm_resource_manager *man, 423 u64 *avail, u64 *visible_avail) 424 { 425 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 426 427 mutex_lock(&bman->lock); 428 *avail = bman->mm.avail >> PAGE_SHIFT; 429 *visible_avail = bman->visible_avail; 430 mutex_unlock(&bman->lock); 431 } 432 433 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 434 void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man, 435 u64 size) 436 { 437 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 438 439 bman->visible_size = size; 440 } 441 #endif 442