1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include "xe_migrate.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/sizes.h> 10 11 #include <drm/drm_managed.h> 12 #include <drm/ttm/ttm_tt.h> 13 #include <drm/xe_drm.h> 14 15 #include <generated/xe_wa_oob.h> 16 17 #include "instructions/xe_gpu_commands.h" 18 #include "instructions/xe_mi_commands.h" 19 #include "regs/xe_gtt_defs.h" 20 #include "tests/xe_test.h" 21 #include "xe_assert.h" 22 #include "xe_bb.h" 23 #include "xe_bo.h" 24 #include "xe_exec_queue.h" 25 #include "xe_ggtt.h" 26 #include "xe_gt.h" 27 #include "xe_hw_engine.h" 28 #include "xe_lrc.h" 29 #include "xe_map.h" 30 #include "xe_mocs.h" 31 #include "xe_pt.h" 32 #include "xe_res_cursor.h" 33 #include "xe_sched_job.h" 34 #include "xe_sync.h" 35 #include "xe_trace_bo.h" 36 #include "xe_vm.h" 37 38 /** 39 * struct xe_migrate - migrate context. 40 */ 41 struct xe_migrate { 42 /** @q: Default exec queue used for migration */ 43 struct xe_exec_queue *q; 44 /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */ 45 struct xe_tile *tile; 46 /** @job_mutex: Timeline mutex for @eng. */ 47 struct mutex job_mutex; 48 /** @pt_bo: Page-table buffer object. */ 49 struct xe_bo *pt_bo; 50 /** @batch_base_ofs: VM offset of the migration batch buffer */ 51 u64 batch_base_ofs; 52 /** @usm_batch_base_ofs: VM offset of the usm batch buffer */ 53 u64 usm_batch_base_ofs; 54 /** @cleared_mem_ofs: VM offset of @cleared_bo. */ 55 u64 cleared_mem_ofs; 56 /** 57 * @fence: dma-fence representing the last migration job batch. 58 * Protected by @job_mutex. 59 */ 60 struct dma_fence *fence; 61 /** 62 * @vm_update_sa: For integrated, used to suballocate page-tables 63 * out of the pt_bo. 64 */ 65 struct drm_suballoc_manager vm_update_sa; 66 /** @min_chunk_size: For dgfx, Minimum chunk size */ 67 u64 min_chunk_size; 68 }; 69 70 #define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ 71 #define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */ 72 #define NUM_KERNEL_PDE 15 73 #define NUM_PT_SLOTS 32 74 #define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M 75 #define MAX_NUM_PTE 512 76 77 /* 78 * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest 79 * legal value accepted. Since that instruction field is always stored in 80 * (val-2) format, this translates to 0x400 dwords for the true maximum length 81 * of the instruction. Subtracting the instruction header (1 dword) and 82 * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values. 83 */ 84 #define MAX_PTE_PER_SDI 0x1FE 85 86 /** 87 * xe_tile_migrate_engine() - Get this tile's migrate engine. 88 * @tile: The tile. 89 * 90 * Returns the default migrate engine of this tile. 91 * TODO: Perhaps this function is slightly misplaced, and even unneeded? 92 * 93 * Return: The default migrate engine 94 */ 95 struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile) 96 { 97 return tile->migrate->q; 98 } 99 100 static void xe_migrate_fini(struct drm_device *dev, void *arg) 101 { 102 struct xe_migrate *m = arg; 103 104 xe_vm_lock(m->q->vm, false); 105 xe_bo_unpin(m->pt_bo); 106 xe_vm_unlock(m->q->vm); 107 108 dma_fence_put(m->fence); 109 xe_bo_put(m->pt_bo); 110 drm_suballoc_manager_fini(&m->vm_update_sa); 111 mutex_destroy(&m->job_mutex); 112 xe_vm_close_and_put(m->q->vm); 113 xe_exec_queue_put(m->q); 114 } 115 116 static u64 xe_migrate_vm_addr(u64 slot, u32 level) 117 { 118 XE_WARN_ON(slot >= NUM_PT_SLOTS); 119 120 /* First slot is reserved for mapping of PT bo and bb, start from 1 */ 121 return (slot + 1ULL) << xe_pt_shift(level + 1); 122 } 123 124 static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr) 125 { 126 /* 127 * Remove the DPA to get a correct offset into identity table for the 128 * migrate offset 129 */ 130 addr -= xe->mem.vram.dpa_base; 131 return addr + (256ULL << xe_pt_shift(2)); 132 } 133 134 static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, 135 struct xe_vm *vm) 136 { 137 struct xe_device *xe = tile_to_xe(tile); 138 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 139 u8 id = tile->id; 140 u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level, 141 num_setup = num_level + 1; 142 u32 map_ofs, level, i; 143 struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo; 144 u64 entry, pt30_ofs; 145 146 /* Can't bump NUM_PT_SLOTS too high */ 147 BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE); 148 /* Must be a multiple of 64K to support all platforms */ 149 BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K); 150 /* And one slot reserved for the 4KiB page table updates */ 151 BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1)); 152 153 /* Need to be sure everything fits in the first PT, or create more */ 154 xe_tile_assert(tile, m->batch_base_ofs + batch->size < SZ_2M); 155 156 bo = xe_bo_create_pin_map(vm->xe, tile, vm, 157 num_entries * XE_PAGE_SIZE, 158 ttm_bo_type_kernel, 159 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 160 XE_BO_FLAG_PINNED); 161 if (IS_ERR(bo)) 162 return PTR_ERR(bo); 163 164 /* PT31 reserved for 2M identity map */ 165 pt30_ofs = bo->size - 2 * XE_PAGE_SIZE; 166 entry = vm->pt_ops->pde_encode_bo(bo, pt30_ofs, pat_index); 167 xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); 168 169 map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE; 170 171 /* Map the entire BO in our level 0 pt */ 172 for (i = 0, level = 0; i < num_entries; level++) { 173 entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE, 174 pat_index, 0); 175 176 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry); 177 178 if (vm->flags & XE_VM_FLAG_64K) 179 i += 16; 180 else 181 i += 1; 182 } 183 184 if (!IS_DGFX(xe)) { 185 /* Write out batch too */ 186 m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE; 187 for (i = 0; i < batch->size; 188 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 189 XE_PAGE_SIZE) { 190 entry = vm->pt_ops->pte_encode_bo(batch, i, 191 pat_index, 0); 192 193 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 194 entry); 195 level++; 196 } 197 if (xe->info.has_usm) { 198 xe_tile_assert(tile, batch->size == SZ_1M); 199 200 batch = tile->primary_gt->usm.bb_pool->bo; 201 m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M; 202 xe_tile_assert(tile, batch->size == SZ_512K); 203 204 for (i = 0; i < batch->size; 205 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 206 XE_PAGE_SIZE) { 207 entry = vm->pt_ops->pte_encode_bo(batch, i, 208 pat_index, 0); 209 210 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 211 entry); 212 level++; 213 } 214 } 215 } else { 216 u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 217 218 m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr); 219 220 if (xe->info.has_usm) { 221 batch = tile->primary_gt->usm.bb_pool->bo; 222 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 223 m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr); 224 } 225 } 226 227 for (level = 1; level < num_level; level++) { 228 u32 flags = 0; 229 230 if (vm->flags & XE_VM_FLAG_64K && level == 1) 231 flags = XE_PDE_64K; 232 233 entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) * 234 XE_PAGE_SIZE, pat_index); 235 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64, 236 entry | flags); 237 } 238 239 /* Write PDE's that point to our BO. */ 240 for (i = 0; i < map_ofs / PAGE_SIZE; i++) { 241 entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE, 242 pat_index); 243 244 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE + 245 (i + 1) * 8, u64, entry); 246 } 247 248 /* Set up a 1GiB NULL mapping at 255GiB offset. */ 249 level = 2; 250 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64, 251 vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) 252 | XE_PTE_NULL); 253 m->cleared_mem_ofs = (255ULL << xe_pt_shift(level)); 254 255 /* Identity map the entire vram at 256GiB offset */ 256 if (IS_DGFX(xe)) { 257 u64 pos, ofs, flags; 258 /* XXX: Unclear if this should be usable_size? */ 259 u64 vram_limit = xe->mem.vram.actual_physical_size + 260 xe->mem.vram.dpa_base; 261 262 level = 2; 263 ofs = map_ofs + XE_PAGE_SIZE * level + 256 * 8; 264 flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, 265 true, 0); 266 267 xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M)); 268 269 /* 270 * Use 1GB pages when possible, last chunk always use 2M 271 * pages as mixing reserved memory (stolen, WOCPM) with a single 272 * mapping is not allowed on certain platforms. 273 */ 274 for (pos = xe->mem.vram.dpa_base; pos < vram_limit; 275 pos += SZ_1G, ofs += 8) { 276 if (pos + SZ_1G >= vram_limit) { 277 u64 pt31_ofs = bo->size - XE_PAGE_SIZE; 278 279 entry = vm->pt_ops->pde_encode_bo(bo, pt31_ofs, 280 pat_index); 281 xe_map_wr(xe, &bo->vmap, ofs, u64, entry); 282 283 flags = vm->pt_ops->pte_encode_addr(xe, 0, 284 pat_index, 285 level - 1, 286 true, 0); 287 288 for (ofs = pt31_ofs; pos < vram_limit; 289 pos += SZ_2M, ofs += 8) 290 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 291 break; /* Ensure pos == vram_limit assert correct */ 292 } 293 294 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 295 } 296 297 xe_assert(xe, pos == vram_limit); 298 } 299 300 /* 301 * Example layout created above, with root level = 3: 302 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's 303 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's 304 * [PT9...PT27]: Userspace PT's for VM_BIND, 4 KiB PTE's 305 * [PT28 = PDE 0] [PT29 = PDE 1] [PT30 = PDE 2] [PT31 = 2M vram identity map] 306 * 307 * This makes the lowest part of the VM point to the pagetables. 308 * Hence the lowest 2M in the vm should point to itself, with a few writes 309 * and flushes, other parts of the VM can be used either for copying and 310 * clearing. 311 * 312 * For performance, the kernel reserves PDE's, so about 20 are left 313 * for async VM updates. 314 * 315 * To make it easier to work, each scratch PT is put in slot (1 + PT #) 316 * everywhere, this allows lockless updates to scratch pages by using 317 * the different addresses in VM. 318 */ 319 #define NUM_VMUSA_UNIT_PER_PAGE 32 320 #define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE) 321 #define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64)) 322 drm_suballoc_manager_init(&m->vm_update_sa, 323 (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) * 324 NUM_VMUSA_UNIT_PER_PAGE, 0); 325 326 m->pt_bo = bo; 327 return 0; 328 } 329 330 /* 331 * Including the reserved copy engine is required to avoid deadlocks due to 332 * migrate jobs servicing the faults gets stuck behind the job that faulted. 333 */ 334 static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt) 335 { 336 u32 logical_mask = 0; 337 struct xe_hw_engine *hwe; 338 enum xe_hw_engine_id id; 339 340 for_each_hw_engine(hwe, gt, id) { 341 if (hwe->class != XE_ENGINE_CLASS_COPY) 342 continue; 343 344 if (xe_gt_is_usm_hwe(gt, hwe)) 345 logical_mask |= BIT(hwe->logical_instance); 346 } 347 348 return logical_mask; 349 } 350 351 /** 352 * xe_migrate_init() - Initialize a migrate context 353 * @tile: Back-pointer to the tile we're initializing for. 354 * 355 * Return: Pointer to a migrate context on success. Error pointer on error. 356 */ 357 struct xe_migrate *xe_migrate_init(struct xe_tile *tile) 358 { 359 struct xe_device *xe = tile_to_xe(tile); 360 struct xe_gt *primary_gt = tile->primary_gt; 361 struct xe_migrate *m; 362 struct xe_vm *vm; 363 int err; 364 365 m = drmm_kzalloc(&xe->drm, sizeof(*m), GFP_KERNEL); 366 if (!m) 367 return ERR_PTR(-ENOMEM); 368 369 m->tile = tile; 370 371 /* Special layout, prepared below.. */ 372 vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | 373 XE_VM_FLAG_SET_TILE_ID(tile)); 374 if (IS_ERR(vm)) 375 return ERR_CAST(vm); 376 377 xe_vm_lock(vm, false); 378 err = xe_migrate_prepare_vm(tile, m, vm); 379 xe_vm_unlock(vm); 380 if (err) { 381 xe_vm_close_and_put(vm); 382 return ERR_PTR(err); 383 } 384 385 if (xe->info.has_usm) { 386 struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt, 387 XE_ENGINE_CLASS_COPY, 388 primary_gt->usm.reserved_bcs_instance, 389 false); 390 u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt); 391 392 if (!hwe || !logical_mask) 393 return ERR_PTR(-EINVAL); 394 395 /* 396 * XXX: Currently only reserving 1 (likely slow) BCS instance on 397 * PVC, may want to revisit if performance is needed. 398 */ 399 m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe, 400 EXEC_QUEUE_FLAG_KERNEL | 401 EXEC_QUEUE_FLAG_PERMANENT | 402 EXEC_QUEUE_FLAG_HIGH_PRIORITY, 0); 403 } else { 404 m->q = xe_exec_queue_create_class(xe, primary_gt, vm, 405 XE_ENGINE_CLASS_COPY, 406 EXEC_QUEUE_FLAG_KERNEL | 407 EXEC_QUEUE_FLAG_PERMANENT); 408 } 409 if (IS_ERR(m->q)) { 410 xe_vm_close_and_put(vm); 411 return ERR_CAST(m->q); 412 } 413 414 mutex_init(&m->job_mutex); 415 fs_reclaim_acquire(GFP_KERNEL); 416 might_lock(&m->job_mutex); 417 fs_reclaim_release(GFP_KERNEL); 418 419 err = drmm_add_action_or_reset(&xe->drm, xe_migrate_fini, m); 420 if (err) 421 return ERR_PTR(err); 422 423 if (IS_DGFX(xe)) { 424 if (xe_device_has_flat_ccs(xe)) 425 /* min chunk size corresponds to 4K of CCS Metadata */ 426 m->min_chunk_size = SZ_4K * SZ_64K / 427 xe_device_ccs_bytes(xe, SZ_64K); 428 else 429 /* Somewhat arbitrary to avoid a huge amount of blits */ 430 m->min_chunk_size = SZ_64K; 431 m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size); 432 drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n", 433 (unsigned long long)m->min_chunk_size); 434 } 435 436 return m; 437 } 438 439 static u64 max_mem_transfer_per_pass(struct xe_device *xe) 440 { 441 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe)) 442 return MAX_CCS_LIMITED_TRANSFER; 443 444 return MAX_PREEMPTDISABLE_TRANSFER; 445 } 446 447 static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur) 448 { 449 struct xe_device *xe = tile_to_xe(m->tile); 450 u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining); 451 452 if (mem_type_is_vram(cur->mem_type)) { 453 /* 454 * VRAM we want to blit in chunks with sizes aligned to 455 * min_chunk_size in order for the offset to CCS metadata to be 456 * page-aligned. If it's the last chunk it may be smaller. 457 * 458 * Another constraint is that we need to limit the blit to 459 * the VRAM block size, unless size is smaller than 460 * min_chunk_size. 461 */ 462 u64 chunk = max_t(u64, cur->size, m->min_chunk_size); 463 464 size = min_t(u64, size, chunk); 465 if (size > m->min_chunk_size) 466 size = round_down(size, m->min_chunk_size); 467 } 468 469 return size; 470 } 471 472 static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur) 473 { 474 /* If the chunk is not fragmented, allow identity map. */ 475 return cur->size >= size; 476 } 477 478 static u32 pte_update_size(struct xe_migrate *m, 479 bool is_vram, 480 struct ttm_resource *res, 481 struct xe_res_cursor *cur, 482 u64 *L0, u64 *L0_ofs, u32 *L0_pt, 483 u32 cmd_size, u32 pt_ofs, u32 avail_pts) 484 { 485 u32 cmds = 0; 486 487 *L0_pt = pt_ofs; 488 if (is_vram && xe_migrate_allow_identity(*L0, cur)) { 489 /* Offset into identity map. */ 490 *L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile), 491 cur->start + vram_region_gpu_offset(res)); 492 cmds += cmd_size; 493 } else { 494 /* Clip L0 to available size */ 495 u64 size = min(*L0, (u64)avail_pts * SZ_2M); 496 u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT; 497 498 *L0 = size; 499 *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0); 500 501 /* MI_STORE_DATA_IMM */ 502 cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI); 503 504 /* PDE qwords */ 505 cmds += num_4k_pages * 2; 506 507 /* Each chunk has a single blit command */ 508 cmds += cmd_size; 509 } 510 511 return cmds; 512 } 513 514 static void emit_pte(struct xe_migrate *m, 515 struct xe_bb *bb, u32 at_pt, 516 bool is_vram, bool is_comp_pte, 517 struct xe_res_cursor *cur, 518 u32 size, struct ttm_resource *res) 519 { 520 struct xe_device *xe = tile_to_xe(m->tile); 521 struct xe_vm *vm = m->q->vm; 522 u16 pat_index; 523 u32 ptes; 524 u64 ofs = (u64)at_pt * XE_PAGE_SIZE; 525 u64 cur_ofs; 526 527 /* Indirect access needs compression enabled uncached PAT index */ 528 if (GRAPHICS_VERx100(xe) >= 2000) 529 pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] : 530 xe->pat.idx[XE_CACHE_WB]; 531 else 532 pat_index = xe->pat.idx[XE_CACHE_WB]; 533 534 ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE); 535 536 while (ptes) { 537 u32 chunk = min(MAX_PTE_PER_SDI, ptes); 538 539 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 540 bb->cs[bb->len++] = ofs; 541 bb->cs[bb->len++] = 0; 542 543 cur_ofs = ofs; 544 ofs += chunk * 8; 545 ptes -= chunk; 546 547 while (chunk--) { 548 u64 addr, flags = 0; 549 bool devmem = false; 550 551 addr = xe_res_dma(cur) & PAGE_MASK; 552 if (is_vram) { 553 if (vm->flags & XE_VM_FLAG_64K) { 554 u64 va = cur_ofs * XE_PAGE_SIZE / 8; 555 556 xe_assert(xe, (va & (SZ_64K - 1)) == 557 (addr & (SZ_64K - 1))); 558 559 flags |= XE_PTE_PS64; 560 } 561 562 addr += vram_region_gpu_offset(res); 563 devmem = true; 564 } 565 566 addr = vm->pt_ops->pte_encode_addr(m->tile->xe, 567 addr, pat_index, 568 0, devmem, flags); 569 bb->cs[bb->len++] = lower_32_bits(addr); 570 bb->cs[bb->len++] = upper_32_bits(addr); 571 572 xe_res_next(cur, min_t(u32, size, PAGE_SIZE)); 573 cur_ofs += 8; 574 } 575 } 576 } 577 578 #define EMIT_COPY_CCS_DW 5 579 static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, 580 u64 dst_ofs, bool dst_is_indirect, 581 u64 src_ofs, bool src_is_indirect, 582 u32 size) 583 { 584 struct xe_device *xe = gt_to_xe(gt); 585 u32 *cs = bb->cs + bb->len; 586 u32 num_ccs_blks; 587 u32 num_pages; 588 u32 ccs_copy_size; 589 u32 mocs; 590 591 if (GRAPHICS_VERx100(xe) >= 2000) { 592 num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE); 593 xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1)); 594 595 ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1); 596 mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index); 597 598 } else { 599 num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size), 600 NUM_CCS_BYTES_PER_BLOCK); 601 xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1)); 602 603 ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1); 604 mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index); 605 } 606 607 *cs++ = XY_CTRL_SURF_COPY_BLT | 608 (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT | 609 (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT | 610 ccs_copy_size; 611 *cs++ = lower_32_bits(src_ofs); 612 *cs++ = upper_32_bits(src_ofs) | mocs; 613 *cs++ = lower_32_bits(dst_ofs); 614 *cs++ = upper_32_bits(dst_ofs) | mocs; 615 616 bb->len = cs - bb->cs; 617 } 618 619 #define EMIT_COPY_DW 10 620 static void emit_copy(struct xe_gt *gt, struct xe_bb *bb, 621 u64 src_ofs, u64 dst_ofs, unsigned int size, 622 unsigned int pitch) 623 { 624 struct xe_device *xe = gt_to_xe(gt); 625 u32 mocs = 0; 626 u32 tile_y = 0; 627 628 xe_gt_assert(gt, size / pitch <= S16_MAX); 629 xe_gt_assert(gt, pitch / 4 <= S16_MAX); 630 xe_gt_assert(gt, pitch <= U16_MAX); 631 632 if (GRAPHICS_VER(xe) >= 20) 633 mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index); 634 635 if (GRAPHICS_VERx100(xe) >= 1250) 636 tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4; 637 638 bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2); 639 bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs; 640 bb->cs[bb->len++] = 0; 641 bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; 642 bb->cs[bb->len++] = lower_32_bits(dst_ofs); 643 bb->cs[bb->len++] = upper_32_bits(dst_ofs); 644 bb->cs[bb->len++] = 0; 645 bb->cs[bb->len++] = pitch | mocs; 646 bb->cs[bb->len++] = lower_32_bits(src_ofs); 647 bb->cs[bb->len++] = upper_32_bits(src_ofs); 648 } 649 650 static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm) 651 { 652 return usm ? m->usm_batch_base_ofs : m->batch_base_ofs; 653 } 654 655 static u32 xe_migrate_ccs_copy(struct xe_migrate *m, 656 struct xe_bb *bb, 657 u64 src_ofs, bool src_is_indirect, 658 u64 dst_ofs, bool dst_is_indirect, u32 dst_size, 659 u64 ccs_ofs, bool copy_ccs) 660 { 661 struct xe_gt *gt = m->tile->primary_gt; 662 u32 flush_flags = 0; 663 664 if (xe_device_has_flat_ccs(gt_to_xe(gt)) && !copy_ccs && dst_is_indirect) { 665 /* 666 * If the src is already in vram, then it should already 667 * have been cleared by us, or has been populated by the 668 * user. Make sure we copy the CCS aux state as-is. 669 * 670 * Otherwise if the bo doesn't have any CCS metadata attached, 671 * we still need to clear it for security reasons. 672 */ 673 u64 ccs_src_ofs = src_is_indirect ? src_ofs : m->cleared_mem_ofs; 674 675 emit_copy_ccs(gt, bb, 676 dst_ofs, true, 677 ccs_src_ofs, src_is_indirect, dst_size); 678 679 flush_flags = MI_FLUSH_DW_CCS; 680 } else if (copy_ccs) { 681 if (!src_is_indirect) 682 src_ofs = ccs_ofs; 683 else if (!dst_is_indirect) 684 dst_ofs = ccs_ofs; 685 686 xe_gt_assert(gt, src_is_indirect || dst_is_indirect); 687 688 emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs, 689 src_is_indirect, dst_size); 690 if (dst_is_indirect) 691 flush_flags = MI_FLUSH_DW_CCS; 692 } 693 694 return flush_flags; 695 } 696 697 /** 698 * xe_migrate_copy() - Copy content of TTM resources. 699 * @m: The migration context. 700 * @src_bo: The buffer object @src is currently bound to. 701 * @dst_bo: If copying between resources created for the same bo, set this to 702 * the same value as @src_bo. If copying between buffer objects, set it to 703 * the buffer object @dst is currently bound to. 704 * @src: The source TTM resource. 705 * @dst: The dst TTM resource. 706 * @copy_only_ccs: If true copy only CCS metadata 707 * 708 * Copies the contents of @src to @dst: On flat CCS devices, 709 * the CCS metadata is copied as well if needed, or if not present, 710 * the CCS metadata of @dst is cleared for security reasons. 711 * 712 * Return: Pointer to a dma_fence representing the last copy batch, or 713 * an error pointer on failure. If there is a failure, any copy operation 714 * started by the function call has been synced. 715 */ 716 struct dma_fence *xe_migrate_copy(struct xe_migrate *m, 717 struct xe_bo *src_bo, 718 struct xe_bo *dst_bo, 719 struct ttm_resource *src, 720 struct ttm_resource *dst, 721 bool copy_only_ccs) 722 { 723 struct xe_gt *gt = m->tile->primary_gt; 724 struct xe_device *xe = gt_to_xe(gt); 725 struct dma_fence *fence = NULL; 726 u64 size = src_bo->size; 727 struct xe_res_cursor src_it, dst_it, ccs_it; 728 u64 src_L0_ofs, dst_L0_ofs; 729 u32 src_L0_pt, dst_L0_pt; 730 u64 src_L0, dst_L0; 731 int pass = 0; 732 int err; 733 bool src_is_pltt = src->mem_type == XE_PL_TT; 734 bool dst_is_pltt = dst->mem_type == XE_PL_TT; 735 bool src_is_vram = mem_type_is_vram(src->mem_type); 736 bool dst_is_vram = mem_type_is_vram(dst->mem_type); 737 bool copy_ccs = xe_device_has_flat_ccs(xe) && 738 xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo); 739 bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram); 740 741 /* Copying CCS between two different BOs is not supported yet. */ 742 if (XE_WARN_ON(copy_ccs && src_bo != dst_bo)) 743 return ERR_PTR(-EINVAL); 744 745 if (src_bo != dst_bo && XE_WARN_ON(src_bo->size != dst_bo->size)) 746 return ERR_PTR(-EINVAL); 747 748 if (!src_is_vram) 749 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); 750 else 751 xe_res_first(src, 0, size, &src_it); 752 if (!dst_is_vram) 753 xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it); 754 else 755 xe_res_first(dst, 0, size, &dst_it); 756 757 if (copy_system_ccs) 758 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), 759 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), 760 &ccs_it); 761 762 while (size) { 763 u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */ 764 struct xe_sched_job *job; 765 struct xe_bb *bb; 766 u32 flush_flags; 767 u32 update_idx; 768 u64 ccs_ofs, ccs_size; 769 u32 ccs_pt; 770 771 bool usm = xe->info.has_usm; 772 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 773 774 src_L0 = xe_migrate_res_sizes(m, &src_it); 775 dst_L0 = xe_migrate_res_sizes(m, &dst_it); 776 777 drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n", 778 pass++, src_L0, dst_L0); 779 780 src_L0 = min(src_L0, dst_L0); 781 782 batch_size += pte_update_size(m, src_is_vram, src, &src_it, &src_L0, 783 &src_L0_ofs, &src_L0_pt, 0, 0, 784 avail_pts); 785 786 batch_size += pte_update_size(m, dst_is_vram, dst, &dst_it, &src_L0, 787 &dst_L0_ofs, &dst_L0_pt, 0, 788 avail_pts, avail_pts); 789 790 if (copy_system_ccs) { 791 ccs_size = xe_device_ccs_bytes(xe, src_L0); 792 batch_size += pte_update_size(m, false, NULL, &ccs_it, &ccs_size, 793 &ccs_ofs, &ccs_pt, 0, 794 2 * avail_pts, 795 avail_pts); 796 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); 797 } 798 799 /* Add copy commands size here */ 800 batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + 801 ((xe_device_has_flat_ccs(xe) ? EMIT_COPY_CCS_DW : 0)); 802 803 bb = xe_bb_new(gt, batch_size, usm); 804 if (IS_ERR(bb)) { 805 err = PTR_ERR(bb); 806 goto err_sync; 807 } 808 809 if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it)) 810 xe_res_next(&src_it, src_L0); 811 else 812 emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs, 813 &src_it, src_L0, src); 814 815 if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it)) 816 xe_res_next(&dst_it, src_L0); 817 else 818 emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs, 819 &dst_it, src_L0, dst); 820 821 if (copy_system_ccs) 822 emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); 823 824 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 825 update_idx = bb->len; 826 827 if (!copy_only_ccs) 828 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE); 829 830 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, 831 IS_DGFX(xe) ? src_is_vram : src_is_pltt, 832 dst_L0_ofs, 833 IS_DGFX(xe) ? dst_is_vram : dst_is_pltt, 834 src_L0, ccs_ofs, copy_ccs); 835 836 job = xe_bb_create_migration_job(m->q, bb, 837 xe_migrate_batch_base(m, usm), 838 update_idx); 839 if (IS_ERR(job)) { 840 err = PTR_ERR(job); 841 goto err; 842 } 843 844 xe_sched_job_add_migrate_flush(job, flush_flags); 845 if (!fence) { 846 err = xe_sched_job_add_deps(job, src_bo->ttm.base.resv, 847 DMA_RESV_USAGE_BOOKKEEP); 848 if (!err && src_bo != dst_bo) 849 err = xe_sched_job_add_deps(job, dst_bo->ttm.base.resv, 850 DMA_RESV_USAGE_BOOKKEEP); 851 if (err) 852 goto err_job; 853 } 854 855 mutex_lock(&m->job_mutex); 856 xe_sched_job_arm(job); 857 dma_fence_put(fence); 858 fence = dma_fence_get(&job->drm.s_fence->finished); 859 xe_sched_job_push(job); 860 861 dma_fence_put(m->fence); 862 m->fence = dma_fence_get(fence); 863 864 mutex_unlock(&m->job_mutex); 865 866 xe_bb_free(bb, fence); 867 size -= src_L0; 868 continue; 869 870 err_job: 871 xe_sched_job_put(job); 872 err: 873 xe_bb_free(bb, NULL); 874 875 err_sync: 876 /* Sync partial copy if any. FIXME: under job_mutex? */ 877 if (fence) { 878 dma_fence_wait(fence, false); 879 dma_fence_put(fence); 880 } 881 882 return ERR_PTR(err); 883 } 884 885 return fence; 886 } 887 888 static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 889 u32 size, u32 pitch) 890 { 891 struct xe_device *xe = gt_to_xe(gt); 892 u32 *cs = bb->cs + bb->len; 893 u32 len = PVC_MEM_SET_CMD_LEN_DW; 894 895 *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2); 896 *cs++ = pitch - 1; 897 *cs++ = (size / pitch) - 1; 898 *cs++ = pitch - 1; 899 *cs++ = lower_32_bits(src_ofs); 900 *cs++ = upper_32_bits(src_ofs); 901 if (GRAPHICS_VERx100(xe) >= 2000) 902 *cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 903 else 904 *cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 905 906 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 907 908 bb->len += len; 909 } 910 911 static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, 912 u64 src_ofs, u32 size, u32 pitch, bool is_vram) 913 { 914 struct xe_device *xe = gt_to_xe(gt); 915 u32 *cs = bb->cs + bb->len; 916 u32 len = XY_FAST_COLOR_BLT_DW; 917 918 if (GRAPHICS_VERx100(xe) < 1250) 919 len = 11; 920 921 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | 922 (len - 2); 923 if (GRAPHICS_VERx100(xe) >= 2000) 924 *cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) | 925 (pitch - 1); 926 else 927 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) | 928 (pitch - 1); 929 *cs++ = 0; 930 *cs++ = (size / pitch) << 16 | pitch / 4; 931 *cs++ = lower_32_bits(src_ofs); 932 *cs++ = upper_32_bits(src_ofs); 933 *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; 934 *cs++ = 0; 935 *cs++ = 0; 936 *cs++ = 0; 937 *cs++ = 0; 938 939 if (len > 11) { 940 *cs++ = 0; 941 *cs++ = 0; 942 *cs++ = 0; 943 *cs++ = 0; 944 *cs++ = 0; 945 } 946 947 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 948 949 bb->len += len; 950 } 951 952 static bool has_service_copy_support(struct xe_gt *gt) 953 { 954 /* 955 * What we care about is whether the architecture was designed with 956 * service copy functionality (specifically the new MEM_SET / MEM_COPY 957 * instructions) so check the architectural engine list rather than the 958 * actual list since these instructions are usable on BCS0 even if 959 * all of the actual service copy engines (BCS1-BCS8) have been fused 960 * off. 961 */ 962 return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8, 963 XE_HW_ENGINE_BCS1); 964 } 965 966 static u32 emit_clear_cmd_len(struct xe_gt *gt) 967 { 968 if (has_service_copy_support(gt)) 969 return PVC_MEM_SET_CMD_LEN_DW; 970 else 971 return XY_FAST_COLOR_BLT_DW; 972 } 973 974 static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 975 u32 size, u32 pitch, bool is_vram) 976 { 977 if (has_service_copy_support(gt)) 978 emit_clear_link_copy(gt, bb, src_ofs, size, pitch); 979 else 980 emit_clear_main_copy(gt, bb, src_ofs, size, pitch, 981 is_vram); 982 } 983 984 /** 985 * xe_migrate_clear() - Copy content of TTM resources. 986 * @m: The migration context. 987 * @bo: The buffer object @dst is currently bound to. 988 * @dst: The dst TTM resource to be cleared. 989 * 990 * Clear the contents of @dst to zero. On flat CCS devices, 991 * the CCS metadata is cleared to zero as well on VRAM destinations. 992 * TODO: Eliminate the @bo argument. 993 * 994 * Return: Pointer to a dma_fence representing the last clear batch, or 995 * an error pointer on failure. If there is a failure, any clear operation 996 * started by the function call has been synced. 997 */ 998 struct dma_fence *xe_migrate_clear(struct xe_migrate *m, 999 struct xe_bo *bo, 1000 struct ttm_resource *dst) 1001 { 1002 bool clear_vram = mem_type_is_vram(dst->mem_type); 1003 struct xe_gt *gt = m->tile->primary_gt; 1004 struct xe_device *xe = gt_to_xe(gt); 1005 bool clear_system_ccs = (xe_bo_needs_ccs_pages(bo) && !IS_DGFX(xe)) ? true : false; 1006 struct dma_fence *fence = NULL; 1007 u64 size = bo->size; 1008 struct xe_res_cursor src_it; 1009 struct ttm_resource *src = dst; 1010 int err; 1011 1012 if (!clear_vram) 1013 xe_res_first_sg(xe_bo_sg(bo), 0, bo->size, &src_it); 1014 else 1015 xe_res_first(src, 0, bo->size, &src_it); 1016 1017 while (size) { 1018 u64 clear_L0_ofs; 1019 u32 clear_L0_pt; 1020 u32 flush_flags = 0; 1021 u64 clear_L0; 1022 struct xe_sched_job *job; 1023 struct xe_bb *bb; 1024 u32 batch_size, update_idx; 1025 1026 bool usm = xe->info.has_usm; 1027 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 1028 1029 clear_L0 = xe_migrate_res_sizes(m, &src_it); 1030 1031 /* Calculate final sizes and batch size.. */ 1032 batch_size = 2 + 1033 pte_update_size(m, clear_vram, src, &src_it, 1034 &clear_L0, &clear_L0_ofs, &clear_L0_pt, 1035 clear_system_ccs ? 0 : emit_clear_cmd_len(gt), 0, 1036 avail_pts); 1037 1038 if (xe_device_has_flat_ccs(xe)) 1039 batch_size += EMIT_COPY_CCS_DW; 1040 1041 /* Clear commands */ 1042 1043 if (WARN_ON_ONCE(!clear_L0)) 1044 break; 1045 1046 bb = xe_bb_new(gt, batch_size, usm); 1047 if (IS_ERR(bb)) { 1048 err = PTR_ERR(bb); 1049 goto err_sync; 1050 } 1051 1052 size -= clear_L0; 1053 /* Preemption is enabled again by the ring ops. */ 1054 if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it)) 1055 xe_res_next(&src_it, clear_L0); 1056 else 1057 emit_pte(m, bb, clear_L0_pt, clear_vram, clear_system_ccs, 1058 &src_it, clear_L0, dst); 1059 1060 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1061 update_idx = bb->len; 1062 1063 if (!clear_system_ccs) 1064 emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram); 1065 1066 if (xe_device_has_flat_ccs(xe)) { 1067 emit_copy_ccs(gt, bb, clear_L0_ofs, true, 1068 m->cleared_mem_ofs, false, clear_L0); 1069 flush_flags = MI_FLUSH_DW_CCS; 1070 } 1071 1072 job = xe_bb_create_migration_job(m->q, bb, 1073 xe_migrate_batch_base(m, usm), 1074 update_idx); 1075 if (IS_ERR(job)) { 1076 err = PTR_ERR(job); 1077 goto err; 1078 } 1079 1080 xe_sched_job_add_migrate_flush(job, flush_flags); 1081 if (!fence) { 1082 /* 1083 * There can't be anything userspace related at this 1084 * point, so we just need to respect any potential move 1085 * fences, which are always tracked as 1086 * DMA_RESV_USAGE_KERNEL. 1087 */ 1088 err = xe_sched_job_add_deps(job, bo->ttm.base.resv, 1089 DMA_RESV_USAGE_KERNEL); 1090 if (err) 1091 goto err_job; 1092 } 1093 1094 mutex_lock(&m->job_mutex); 1095 xe_sched_job_arm(job); 1096 dma_fence_put(fence); 1097 fence = dma_fence_get(&job->drm.s_fence->finished); 1098 xe_sched_job_push(job); 1099 1100 dma_fence_put(m->fence); 1101 m->fence = dma_fence_get(fence); 1102 1103 mutex_unlock(&m->job_mutex); 1104 1105 xe_bb_free(bb, fence); 1106 continue; 1107 1108 err_job: 1109 xe_sched_job_put(job); 1110 err: 1111 xe_bb_free(bb, NULL); 1112 err_sync: 1113 /* Sync partial copies if any. FIXME: job_mutex? */ 1114 if (fence) { 1115 dma_fence_wait(m->fence, false); 1116 dma_fence_put(fence); 1117 } 1118 1119 return ERR_PTR(err); 1120 } 1121 1122 if (clear_system_ccs) 1123 bo->ccs_cleared = true; 1124 1125 return fence; 1126 } 1127 1128 static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs, 1129 const struct xe_vm_pgtable_update *update, 1130 struct xe_migrate_pt_update *pt_update) 1131 { 1132 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1133 u32 chunk; 1134 u32 ofs = update->ofs, size = update->qwords; 1135 1136 /* 1137 * If we have 512 entries (max), we would populate it ourselves, 1138 * and update the PDE above it to the new pointer. 1139 * The only time this can only happen if we have to update the top 1140 * PDE. This requires a BO that is almost vm->size big. 1141 * 1142 * This shouldn't be possible in practice.. might change when 16K 1143 * pages are used. Hence the assert. 1144 */ 1145 xe_tile_assert(tile, update->qwords < MAX_NUM_PTE); 1146 if (!ppgtt_ofs) 1147 ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile), 1148 xe_bo_addr(update->pt_bo, 0, 1149 XE_PAGE_SIZE)); 1150 1151 do { 1152 u64 addr = ppgtt_ofs + ofs * 8; 1153 1154 chunk = min(size, MAX_PTE_PER_SDI); 1155 1156 /* Ensure populatefn can do memset64 by aligning bb->cs */ 1157 if (!(bb->len & 1)) 1158 bb->cs[bb->len++] = MI_NOOP; 1159 1160 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 1161 bb->cs[bb->len++] = lower_32_bits(addr); 1162 bb->cs[bb->len++] = upper_32_bits(addr); 1163 ops->populate(pt_update, tile, NULL, bb->cs + bb->len, ofs, chunk, 1164 update); 1165 1166 bb->len += chunk * 2; 1167 ofs += chunk; 1168 size -= chunk; 1169 } while (size); 1170 } 1171 1172 struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) 1173 { 1174 return xe_vm_get(m->q->vm); 1175 } 1176 1177 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1178 struct migrate_test_params { 1179 struct xe_test_priv base; 1180 bool force_gpu; 1181 }; 1182 1183 #define to_migrate_test_params(_priv) \ 1184 container_of(_priv, struct migrate_test_params, base) 1185 #endif 1186 1187 static struct dma_fence * 1188 xe_migrate_update_pgtables_cpu(struct xe_migrate *m, 1189 struct xe_vm *vm, struct xe_bo *bo, 1190 const struct xe_vm_pgtable_update *updates, 1191 u32 num_updates, bool wait_vm, 1192 struct xe_migrate_pt_update *pt_update) 1193 { 1194 XE_TEST_DECLARE(struct migrate_test_params *test = 1195 to_migrate_test_params 1196 (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));) 1197 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1198 struct dma_fence *fence; 1199 int err; 1200 u32 i; 1201 1202 if (XE_TEST_ONLY(test && test->force_gpu)) 1203 return ERR_PTR(-ETIME); 1204 1205 if (bo && !dma_resv_test_signaled(bo->ttm.base.resv, 1206 DMA_RESV_USAGE_KERNEL)) 1207 return ERR_PTR(-ETIME); 1208 1209 if (wait_vm && !dma_resv_test_signaled(xe_vm_resv(vm), 1210 DMA_RESV_USAGE_BOOKKEEP)) 1211 return ERR_PTR(-ETIME); 1212 1213 if (ops->pre_commit) { 1214 pt_update->job = NULL; 1215 err = ops->pre_commit(pt_update); 1216 if (err) 1217 return ERR_PTR(err); 1218 } 1219 for (i = 0; i < num_updates; i++) { 1220 const struct xe_vm_pgtable_update *update = &updates[i]; 1221 1222 ops->populate(pt_update, m->tile, &update->pt_bo->vmap, NULL, 1223 update->ofs, update->qwords, update); 1224 } 1225 1226 if (vm) { 1227 trace_xe_vm_cpu_bind(vm); 1228 xe_device_wmb(vm->xe); 1229 } 1230 1231 fence = dma_fence_get_stub(); 1232 1233 return fence; 1234 } 1235 1236 static bool no_in_syncs(struct xe_vm *vm, struct xe_exec_queue *q, 1237 struct xe_sync_entry *syncs, u32 num_syncs) 1238 { 1239 struct dma_fence *fence; 1240 int i; 1241 1242 for (i = 0; i < num_syncs; i++) { 1243 fence = syncs[i].fence; 1244 1245 if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 1246 &fence->flags)) 1247 return false; 1248 } 1249 if (q) { 1250 fence = xe_exec_queue_last_fence_get(q, vm); 1251 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { 1252 dma_fence_put(fence); 1253 return false; 1254 } 1255 dma_fence_put(fence); 1256 } 1257 1258 return true; 1259 } 1260 1261 /** 1262 * xe_migrate_update_pgtables() - Pipelined page-table update 1263 * @m: The migrate context. 1264 * @vm: The vm we'll be updating. 1265 * @bo: The bo whose dma-resv we will await before updating, or NULL if userptr. 1266 * @q: The exec queue to be used for the update or NULL if the default 1267 * migration engine is to be used. 1268 * @updates: An array of update descriptors. 1269 * @num_updates: Number of descriptors in @updates. 1270 * @syncs: Array of xe_sync_entry to await before updating. Note that waits 1271 * will block the engine timeline. 1272 * @num_syncs: Number of entries in @syncs. 1273 * @pt_update: Pointer to a struct xe_migrate_pt_update, which contains 1274 * pointers to callback functions and, if subclassed, private arguments to 1275 * those. 1276 * 1277 * Perform a pipelined page-table update. The update descriptors are typically 1278 * built under the same lock critical section as a call to this function. If 1279 * using the default engine for the updates, they will be performed in the 1280 * order they grab the job_mutex. If different engines are used, external 1281 * synchronization is needed for overlapping updates to maintain page-table 1282 * consistency. Note that the meaing of "overlapping" is that the updates 1283 * touch the same page-table, which might be a higher-level page-directory. 1284 * If no pipelining is needed, then updates may be performed by the cpu. 1285 * 1286 * Return: A dma_fence that, when signaled, indicates the update completion. 1287 */ 1288 struct dma_fence * 1289 xe_migrate_update_pgtables(struct xe_migrate *m, 1290 struct xe_vm *vm, 1291 struct xe_bo *bo, 1292 struct xe_exec_queue *q, 1293 const struct xe_vm_pgtable_update *updates, 1294 u32 num_updates, 1295 struct xe_sync_entry *syncs, u32 num_syncs, 1296 struct xe_migrate_pt_update *pt_update) 1297 { 1298 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1299 struct xe_tile *tile = m->tile; 1300 struct xe_gt *gt = tile->primary_gt; 1301 struct xe_device *xe = tile_to_xe(tile); 1302 struct xe_sched_job *job; 1303 struct dma_fence *fence; 1304 struct drm_suballoc *sa_bo = NULL; 1305 struct xe_vma *vma = pt_update->vma; 1306 struct xe_bb *bb; 1307 u32 i, batch_size, ppgtt_ofs, update_idx, page_ofs = 0; 1308 u64 addr; 1309 int err = 0; 1310 bool usm = !q && xe->info.has_usm; 1311 bool first_munmap_rebind = vma && 1312 vma->gpuva.flags & XE_VMA_FIRST_REBIND; 1313 struct xe_exec_queue *q_override = !q ? m->q : q; 1314 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 1315 1316 /* Use the CPU if no in syncs and engine is idle */ 1317 if (no_in_syncs(vm, q, syncs, num_syncs) && xe_exec_queue_is_idle(q_override)) { 1318 fence = xe_migrate_update_pgtables_cpu(m, vm, bo, updates, 1319 num_updates, 1320 first_munmap_rebind, 1321 pt_update); 1322 if (!IS_ERR(fence) || fence == ERR_PTR(-EAGAIN)) 1323 return fence; 1324 } 1325 1326 /* fixed + PTE entries */ 1327 if (IS_DGFX(xe)) 1328 batch_size = 2; 1329 else 1330 batch_size = 6 + num_updates * 2; 1331 1332 for (i = 0; i < num_updates; i++) { 1333 u32 num_cmds = DIV_ROUND_UP(updates[i].qwords, MAX_PTE_PER_SDI); 1334 1335 /* align noop + MI_STORE_DATA_IMM cmd prefix */ 1336 batch_size += 4 * num_cmds + updates[i].qwords * 2; 1337 } 1338 1339 /* 1340 * XXX: Create temp bo to copy from, if batch_size becomes too big? 1341 * 1342 * Worst case: Sum(2 * (each lower level page size) + (top level page size)) 1343 * Should be reasonably bound.. 1344 */ 1345 xe_tile_assert(tile, batch_size < SZ_128K); 1346 1347 bb = xe_bb_new(gt, batch_size, !q && xe->info.has_usm); 1348 if (IS_ERR(bb)) 1349 return ERR_CAST(bb); 1350 1351 /* For sysmem PTE's, need to map them in our hole.. */ 1352 if (!IS_DGFX(xe)) { 1353 ppgtt_ofs = NUM_KERNEL_PDE - 1; 1354 if (q) { 1355 xe_tile_assert(tile, num_updates <= NUM_VMUSA_WRITES_PER_UNIT); 1356 1357 sa_bo = drm_suballoc_new(&m->vm_update_sa, 1, 1358 GFP_KERNEL, true, 0); 1359 if (IS_ERR(sa_bo)) { 1360 err = PTR_ERR(sa_bo); 1361 goto err_bb; 1362 } 1363 1364 ppgtt_ofs = NUM_KERNEL_PDE + 1365 (drm_suballoc_soffset(sa_bo) / 1366 NUM_VMUSA_UNIT_PER_PAGE); 1367 page_ofs = (drm_suballoc_soffset(sa_bo) % 1368 NUM_VMUSA_UNIT_PER_PAGE) * 1369 VM_SA_UPDATE_UNIT_SIZE; 1370 } 1371 1372 /* Map our PT's to gtt */ 1373 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(num_updates); 1374 bb->cs[bb->len++] = ppgtt_ofs * XE_PAGE_SIZE + page_ofs; 1375 bb->cs[bb->len++] = 0; /* upper_32_bits */ 1376 1377 for (i = 0; i < num_updates; i++) { 1378 struct xe_bo *pt_bo = updates[i].pt_bo; 1379 1380 xe_tile_assert(tile, pt_bo->size == SZ_4K); 1381 1382 addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, pat_index, 0); 1383 bb->cs[bb->len++] = lower_32_bits(addr); 1384 bb->cs[bb->len++] = upper_32_bits(addr); 1385 } 1386 1387 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1388 update_idx = bb->len; 1389 1390 addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + 1391 (page_ofs / sizeof(u64)) * XE_PAGE_SIZE; 1392 for (i = 0; i < num_updates; i++) 1393 write_pgtable(tile, bb, addr + i * XE_PAGE_SIZE, 1394 &updates[i], pt_update); 1395 } else { 1396 /* phys pages, no preamble required */ 1397 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1398 update_idx = bb->len; 1399 1400 for (i = 0; i < num_updates; i++) 1401 write_pgtable(tile, bb, 0, &updates[i], pt_update); 1402 } 1403 1404 job = xe_bb_create_migration_job(q ?: m->q, bb, 1405 xe_migrate_batch_base(m, usm), 1406 update_idx); 1407 if (IS_ERR(job)) { 1408 err = PTR_ERR(job); 1409 goto err_sa; 1410 } 1411 1412 /* Wait on BO move */ 1413 if (bo) { 1414 err = xe_sched_job_add_deps(job, bo->ttm.base.resv, 1415 DMA_RESV_USAGE_KERNEL); 1416 if (err) 1417 goto err_job; 1418 } 1419 1420 /* 1421 * Munmap style VM unbind, need to wait for all jobs to be complete / 1422 * trigger preempts before moving forward 1423 */ 1424 if (first_munmap_rebind) { 1425 err = xe_sched_job_add_deps(job, xe_vm_resv(vm), 1426 DMA_RESV_USAGE_BOOKKEEP); 1427 if (err) 1428 goto err_job; 1429 } 1430 1431 err = xe_sched_job_last_fence_add_dep(job, vm); 1432 for (i = 0; !err && i < num_syncs; i++) 1433 err = xe_sync_entry_add_deps(&syncs[i], job); 1434 1435 if (err) 1436 goto err_job; 1437 1438 if (ops->pre_commit) { 1439 pt_update->job = job; 1440 err = ops->pre_commit(pt_update); 1441 if (err) 1442 goto err_job; 1443 } 1444 if (!q) 1445 mutex_lock(&m->job_mutex); 1446 1447 xe_sched_job_arm(job); 1448 fence = dma_fence_get(&job->drm.s_fence->finished); 1449 xe_sched_job_push(job); 1450 1451 if (!q) 1452 mutex_unlock(&m->job_mutex); 1453 1454 xe_bb_free(bb, fence); 1455 drm_suballoc_free(sa_bo, fence); 1456 1457 return fence; 1458 1459 err_job: 1460 xe_sched_job_put(job); 1461 err_sa: 1462 drm_suballoc_free(sa_bo, NULL); 1463 err_bb: 1464 xe_bb_free(bb, NULL); 1465 return ERR_PTR(err); 1466 } 1467 1468 /** 1469 * xe_migrate_wait() - Complete all operations using the xe_migrate context 1470 * @m: Migrate context to wait for. 1471 * 1472 * Waits until the GPU no longer uses the migrate context's default engine 1473 * or its page-table objects. FIXME: What about separate page-table update 1474 * engines? 1475 */ 1476 void xe_migrate_wait(struct xe_migrate *m) 1477 { 1478 if (m->fence) 1479 dma_fence_wait(m->fence, false); 1480 } 1481 1482 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1483 #include "tests/xe_migrate.c" 1484 #endif 1485