1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include "xe_migrate.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/sizes.h> 10 11 #include <drm/drm_managed.h> 12 #include <drm/ttm/ttm_tt.h> 13 #include <drm/xe_drm.h> 14 15 #include <generated/xe_wa_oob.h> 16 17 #include "instructions/xe_gpu_commands.h" 18 #include "instructions/xe_mi_commands.h" 19 #include "regs/xe_gtt_defs.h" 20 #include "tests/xe_test.h" 21 #include "xe_assert.h" 22 #include "xe_bb.h" 23 #include "xe_bo.h" 24 #include "xe_exec_queue.h" 25 #include "xe_ggtt.h" 26 #include "xe_gt.h" 27 #include "xe_hw_engine.h" 28 #include "xe_lrc.h" 29 #include "xe_map.h" 30 #include "xe_mocs.h" 31 #include "xe_pt.h" 32 #include "xe_res_cursor.h" 33 #include "xe_sched_job.h" 34 #include "xe_sync.h" 35 #include "xe_trace.h" 36 #include "xe_vm.h" 37 38 /** 39 * struct xe_migrate - migrate context. 40 */ 41 struct xe_migrate { 42 /** @q: Default exec queue used for migration */ 43 struct xe_exec_queue *q; 44 /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */ 45 struct xe_tile *tile; 46 /** @job_mutex: Timeline mutex for @eng. */ 47 struct mutex job_mutex; 48 /** @pt_bo: Page-table buffer object. */ 49 struct xe_bo *pt_bo; 50 /** @batch_base_ofs: VM offset of the migration batch buffer */ 51 u64 batch_base_ofs; 52 /** @usm_batch_base_ofs: VM offset of the usm batch buffer */ 53 u64 usm_batch_base_ofs; 54 /** @cleared_mem_ofs: VM offset of @cleared_bo. */ 55 u64 cleared_mem_ofs; 56 /** 57 * @fence: dma-fence representing the last migration job batch. 58 * Protected by @job_mutex. 59 */ 60 struct dma_fence *fence; 61 /** 62 * @vm_update_sa: For integrated, used to suballocate page-tables 63 * out of the pt_bo. 64 */ 65 struct drm_suballoc_manager vm_update_sa; 66 /** @min_chunk_size: For dgfx, Minimum chunk size */ 67 u64 min_chunk_size; 68 }; 69 70 #define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ 71 #define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */ 72 #define NUM_KERNEL_PDE 15 73 #define NUM_PT_SLOTS 32 74 #define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M 75 #define MAX_NUM_PTE 512 76 77 /* 78 * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest 79 * legal value accepted. Since that instruction field is always stored in 80 * (val-2) format, this translates to 0x400 dwords for the true maximum length 81 * of the instruction. Subtracting the instruction header (1 dword) and 82 * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values. 83 */ 84 #define MAX_PTE_PER_SDI 0x1FE 85 86 /** 87 * xe_tile_migrate_engine() - Get this tile's migrate engine. 88 * @tile: The tile. 89 * 90 * Returns the default migrate engine of this tile. 91 * TODO: Perhaps this function is slightly misplaced, and even unneeded? 92 * 93 * Return: The default migrate engine 94 */ 95 struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile) 96 { 97 return tile->migrate->q; 98 } 99 100 static void xe_migrate_fini(struct drm_device *dev, void *arg) 101 { 102 struct xe_migrate *m = arg; 103 104 xe_vm_lock(m->q->vm, false); 105 xe_bo_unpin(m->pt_bo); 106 xe_vm_unlock(m->q->vm); 107 108 dma_fence_put(m->fence); 109 xe_bo_put(m->pt_bo); 110 drm_suballoc_manager_fini(&m->vm_update_sa); 111 mutex_destroy(&m->job_mutex); 112 xe_vm_close_and_put(m->q->vm); 113 xe_exec_queue_put(m->q); 114 } 115 116 static u64 xe_migrate_vm_addr(u64 slot, u32 level) 117 { 118 XE_WARN_ON(slot >= NUM_PT_SLOTS); 119 120 /* First slot is reserved for mapping of PT bo and bb, start from 1 */ 121 return (slot + 1ULL) << xe_pt_shift(level + 1); 122 } 123 124 static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr) 125 { 126 /* 127 * Remove the DPA to get a correct offset into identity table for the 128 * migrate offset 129 */ 130 addr -= xe->mem.vram.dpa_base; 131 return addr + (256ULL << xe_pt_shift(2)); 132 } 133 134 static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, 135 struct xe_vm *vm) 136 { 137 struct xe_device *xe = tile_to_xe(tile); 138 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 139 u8 id = tile->id; 140 u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level, 141 num_setup = num_level + 1; 142 u32 map_ofs, level, i; 143 struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo; 144 u64 entry, pt30_ofs; 145 146 /* Can't bump NUM_PT_SLOTS too high */ 147 BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE); 148 /* Must be a multiple of 64K to support all platforms */ 149 BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K); 150 /* And one slot reserved for the 4KiB page table updates */ 151 BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1)); 152 153 /* Need to be sure everything fits in the first PT, or create more */ 154 xe_tile_assert(tile, m->batch_base_ofs + batch->size < SZ_2M); 155 156 bo = xe_bo_create_pin_map(vm->xe, tile, vm, 157 num_entries * XE_PAGE_SIZE, 158 ttm_bo_type_kernel, 159 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 160 XE_BO_FLAG_PINNED); 161 if (IS_ERR(bo)) 162 return PTR_ERR(bo); 163 164 /* PT31 reserved for 2M identity map */ 165 pt30_ofs = bo->size - 2 * XE_PAGE_SIZE; 166 entry = vm->pt_ops->pde_encode_bo(bo, pt30_ofs, pat_index); 167 xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); 168 169 map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE; 170 171 /* Map the entire BO in our level 0 pt */ 172 for (i = 0, level = 0; i < num_entries; level++) { 173 entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE, 174 pat_index, 0); 175 176 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry); 177 178 if (vm->flags & XE_VM_FLAG_64K) 179 i += 16; 180 else 181 i += 1; 182 } 183 184 if (!IS_DGFX(xe)) { 185 /* Write out batch too */ 186 m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE; 187 for (i = 0; i < batch->size; 188 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 189 XE_PAGE_SIZE) { 190 entry = vm->pt_ops->pte_encode_bo(batch, i, 191 pat_index, 0); 192 193 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 194 entry); 195 level++; 196 } 197 if (xe->info.has_usm) { 198 xe_tile_assert(tile, batch->size == SZ_1M); 199 200 batch = tile->primary_gt->usm.bb_pool->bo; 201 m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M; 202 xe_tile_assert(tile, batch->size == SZ_512K); 203 204 for (i = 0; i < batch->size; 205 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 206 XE_PAGE_SIZE) { 207 entry = vm->pt_ops->pte_encode_bo(batch, i, 208 pat_index, 0); 209 210 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 211 entry); 212 level++; 213 } 214 } 215 } else { 216 u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 217 218 m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr); 219 220 if (xe->info.has_usm) { 221 batch = tile->primary_gt->usm.bb_pool->bo; 222 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 223 m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr); 224 } 225 } 226 227 for (level = 1; level < num_level; level++) { 228 u32 flags = 0; 229 230 if (vm->flags & XE_VM_FLAG_64K && level == 1) 231 flags = XE_PDE_64K; 232 233 entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) * 234 XE_PAGE_SIZE, pat_index); 235 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64, 236 entry | flags); 237 } 238 239 /* Write PDE's that point to our BO. */ 240 for (i = 0; i < map_ofs / PAGE_SIZE; i++) { 241 entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE, 242 pat_index); 243 244 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE + 245 (i + 1) * 8, u64, entry); 246 } 247 248 /* Set up a 1GiB NULL mapping at 255GiB offset. */ 249 level = 2; 250 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64, 251 vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) 252 | XE_PTE_NULL); 253 m->cleared_mem_ofs = (255ULL << xe_pt_shift(level)); 254 255 /* Identity map the entire vram at 256GiB offset */ 256 if (IS_DGFX(xe)) { 257 u64 pos, ofs, flags; 258 /* XXX: Unclear if this should be usable_size? */ 259 u64 vram_limit = xe->mem.vram.actual_physical_size + 260 xe->mem.vram.dpa_base; 261 262 level = 2; 263 ofs = map_ofs + XE_PAGE_SIZE * level + 256 * 8; 264 flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, 265 true, 0); 266 267 xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M)); 268 269 /* 270 * Use 1GB pages when possible, last chunk always use 2M 271 * pages as mixing reserved memory (stolen, WOCPM) with a single 272 * mapping is not allowed on certain platforms. 273 */ 274 for (pos = xe->mem.vram.dpa_base; pos < vram_limit; 275 pos += SZ_1G, ofs += 8) { 276 if (pos + SZ_1G >= vram_limit) { 277 u64 pt31_ofs = bo->size - XE_PAGE_SIZE; 278 279 entry = vm->pt_ops->pde_encode_bo(bo, pt31_ofs, 280 pat_index); 281 xe_map_wr(xe, &bo->vmap, ofs, u64, entry); 282 283 flags = vm->pt_ops->pte_encode_addr(xe, 0, 284 pat_index, 285 level - 1, 286 true, 0); 287 288 for (ofs = pt31_ofs; pos < vram_limit; 289 pos += SZ_2M, ofs += 8) 290 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 291 break; /* Ensure pos == vram_limit assert correct */ 292 } 293 294 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 295 } 296 297 xe_assert(xe, pos == vram_limit); 298 } 299 300 /* 301 * Example layout created above, with root level = 3: 302 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's 303 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's 304 * [PT9...PT27]: Userspace PT's for VM_BIND, 4 KiB PTE's 305 * [PT28 = PDE 0] [PT29 = PDE 1] [PT30 = PDE 2] [PT31 = 2M vram identity map] 306 * 307 * This makes the lowest part of the VM point to the pagetables. 308 * Hence the lowest 2M in the vm should point to itself, with a few writes 309 * and flushes, other parts of the VM can be used either for copying and 310 * clearing. 311 * 312 * For performance, the kernel reserves PDE's, so about 20 are left 313 * for async VM updates. 314 * 315 * To make it easier to work, each scratch PT is put in slot (1 + PT #) 316 * everywhere, this allows lockless updates to scratch pages by using 317 * the different addresses in VM. 318 */ 319 #define NUM_VMUSA_UNIT_PER_PAGE 32 320 #define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE) 321 #define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64)) 322 drm_suballoc_manager_init(&m->vm_update_sa, 323 (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) * 324 NUM_VMUSA_UNIT_PER_PAGE, 0); 325 326 m->pt_bo = bo; 327 return 0; 328 } 329 330 /* 331 * Including the reserved copy engine is required to avoid deadlocks due to 332 * migrate jobs servicing the faults gets stuck behind the job that faulted. 333 */ 334 static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt) 335 { 336 u32 logical_mask = 0; 337 struct xe_hw_engine *hwe; 338 enum xe_hw_engine_id id; 339 340 for_each_hw_engine(hwe, gt, id) { 341 if (hwe->class != XE_ENGINE_CLASS_COPY) 342 continue; 343 344 if (xe_gt_is_usm_hwe(gt, hwe)) 345 logical_mask |= BIT(hwe->logical_instance); 346 } 347 348 return logical_mask; 349 } 350 351 /** 352 * xe_migrate_init() - Initialize a migrate context 353 * @tile: Back-pointer to the tile we're initializing for. 354 * 355 * Return: Pointer to a migrate context on success. Error pointer on error. 356 */ 357 struct xe_migrate *xe_migrate_init(struct xe_tile *tile) 358 { 359 struct xe_device *xe = tile_to_xe(tile); 360 struct xe_gt *primary_gt = tile->primary_gt; 361 struct xe_migrate *m; 362 struct xe_vm *vm; 363 int err; 364 365 m = drmm_kzalloc(&xe->drm, sizeof(*m), GFP_KERNEL); 366 if (!m) 367 return ERR_PTR(-ENOMEM); 368 369 m->tile = tile; 370 371 /* Special layout, prepared below.. */ 372 vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | 373 XE_VM_FLAG_SET_TILE_ID(tile)); 374 if (IS_ERR(vm)) 375 return ERR_CAST(vm); 376 377 xe_vm_lock(vm, false); 378 err = xe_migrate_prepare_vm(tile, m, vm); 379 xe_vm_unlock(vm); 380 if (err) { 381 xe_vm_close_and_put(vm); 382 return ERR_PTR(err); 383 } 384 385 if (xe->info.has_usm) { 386 struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt, 387 XE_ENGINE_CLASS_COPY, 388 primary_gt->usm.reserved_bcs_instance, 389 false); 390 u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt); 391 392 if (!hwe || !logical_mask) 393 return ERR_PTR(-EINVAL); 394 395 /* 396 * XXX: Currently only reserving 1 (likely slow) BCS instance on 397 * PVC, may want to revisit if performance is needed. 398 */ 399 m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe, 400 EXEC_QUEUE_FLAG_KERNEL | 401 EXEC_QUEUE_FLAG_PERMANENT | 402 EXEC_QUEUE_FLAG_HIGH_PRIORITY, 0); 403 } else { 404 m->q = xe_exec_queue_create_class(xe, primary_gt, vm, 405 XE_ENGINE_CLASS_COPY, 406 EXEC_QUEUE_FLAG_KERNEL | 407 EXEC_QUEUE_FLAG_PERMANENT); 408 } 409 if (IS_ERR(m->q)) { 410 xe_vm_close_and_put(vm); 411 return ERR_CAST(m->q); 412 } 413 414 mutex_init(&m->job_mutex); 415 fs_reclaim_acquire(GFP_KERNEL); 416 might_lock(&m->job_mutex); 417 fs_reclaim_release(GFP_KERNEL); 418 419 err = drmm_add_action_or_reset(&xe->drm, xe_migrate_fini, m); 420 if (err) 421 return ERR_PTR(err); 422 423 if (IS_DGFX(xe)) { 424 if (xe_device_has_flat_ccs(xe)) 425 /* min chunk size corresponds to 4K of CCS Metadata */ 426 m->min_chunk_size = SZ_4K * SZ_64K / 427 xe_device_ccs_bytes(xe, SZ_64K); 428 else 429 /* Somewhat arbitrary to avoid a huge amount of blits */ 430 m->min_chunk_size = SZ_64K; 431 m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size); 432 drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n", 433 (unsigned long long)m->min_chunk_size); 434 } 435 436 return m; 437 } 438 439 static u64 max_mem_transfer_per_pass(struct xe_device *xe) 440 { 441 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe)) 442 return MAX_CCS_LIMITED_TRANSFER; 443 444 return MAX_PREEMPTDISABLE_TRANSFER; 445 } 446 447 static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur) 448 { 449 struct xe_device *xe = tile_to_xe(m->tile); 450 u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining); 451 452 if (mem_type_is_vram(cur->mem_type)) { 453 /* 454 * VRAM we want to blit in chunks with sizes aligned to 455 * min_chunk_size in order for the offset to CCS metadata to be 456 * page-aligned. If it's the last chunk it may be smaller. 457 * 458 * Another constraint is that we need to limit the blit to 459 * the VRAM block size, unless size is smaller than 460 * min_chunk_size. 461 */ 462 u64 chunk = max_t(u64, cur->size, m->min_chunk_size); 463 464 size = min_t(u64, size, chunk); 465 if (size > m->min_chunk_size) 466 size = round_down(size, m->min_chunk_size); 467 } 468 469 return size; 470 } 471 472 static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur) 473 { 474 /* If the chunk is not fragmented, allow identity map. */ 475 return cur->size >= size; 476 } 477 478 static u32 pte_update_size(struct xe_migrate *m, 479 bool is_vram, 480 struct ttm_resource *res, 481 struct xe_res_cursor *cur, 482 u64 *L0, u64 *L0_ofs, u32 *L0_pt, 483 u32 cmd_size, u32 pt_ofs, u32 avail_pts) 484 { 485 u32 cmds = 0; 486 487 *L0_pt = pt_ofs; 488 if (is_vram && xe_migrate_allow_identity(*L0, cur)) { 489 /* Offset into identity map. */ 490 *L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile), 491 cur->start + vram_region_gpu_offset(res)); 492 cmds += cmd_size; 493 } else { 494 /* Clip L0 to available size */ 495 u64 size = min(*L0, (u64)avail_pts * SZ_2M); 496 u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT; 497 498 *L0 = size; 499 *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0); 500 501 /* MI_STORE_DATA_IMM */ 502 cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI); 503 504 /* PDE qwords */ 505 cmds += num_4k_pages * 2; 506 507 /* Each chunk has a single blit command */ 508 cmds += cmd_size; 509 } 510 511 return cmds; 512 } 513 514 static void emit_pte(struct xe_migrate *m, 515 struct xe_bb *bb, u32 at_pt, 516 bool is_vram, bool is_comp_pte, 517 struct xe_res_cursor *cur, 518 u32 size, struct ttm_resource *res) 519 { 520 struct xe_device *xe = tile_to_xe(m->tile); 521 struct xe_vm *vm = m->q->vm; 522 u16 pat_index; 523 u32 ptes; 524 u64 ofs = (u64)at_pt * XE_PAGE_SIZE; 525 u64 cur_ofs; 526 527 /* Indirect access needs compression enabled uncached PAT index */ 528 if (GRAPHICS_VERx100(xe) >= 2000) 529 pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] : 530 xe->pat.idx[XE_CACHE_WB]; 531 else 532 pat_index = xe->pat.idx[XE_CACHE_WB]; 533 534 ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE); 535 536 while (ptes) { 537 u32 chunk = min(MAX_PTE_PER_SDI, ptes); 538 539 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 540 bb->cs[bb->len++] = ofs; 541 bb->cs[bb->len++] = 0; 542 543 cur_ofs = ofs; 544 ofs += chunk * 8; 545 ptes -= chunk; 546 547 while (chunk--) { 548 u64 addr, flags = 0; 549 bool devmem = false; 550 551 addr = xe_res_dma(cur) & PAGE_MASK; 552 if (is_vram) { 553 if (vm->flags & XE_VM_FLAG_64K) { 554 u64 va = cur_ofs * XE_PAGE_SIZE / 8; 555 556 xe_assert(xe, (va & (SZ_64K - 1)) == 557 (addr & (SZ_64K - 1))); 558 559 flags |= XE_PTE_PS64; 560 } 561 562 addr += vram_region_gpu_offset(res); 563 devmem = true; 564 } 565 566 addr = vm->pt_ops->pte_encode_addr(m->tile->xe, 567 addr, pat_index, 568 0, devmem, flags); 569 bb->cs[bb->len++] = lower_32_bits(addr); 570 bb->cs[bb->len++] = upper_32_bits(addr); 571 572 xe_res_next(cur, min_t(u32, size, PAGE_SIZE)); 573 cur_ofs += 8; 574 } 575 } 576 } 577 578 #define EMIT_COPY_CCS_DW 5 579 static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, 580 u64 dst_ofs, bool dst_is_indirect, 581 u64 src_ofs, bool src_is_indirect, 582 u32 size) 583 { 584 struct xe_device *xe = gt_to_xe(gt); 585 u32 *cs = bb->cs + bb->len; 586 u32 num_ccs_blks; 587 u32 num_pages; 588 u32 ccs_copy_size; 589 u32 mocs; 590 591 if (GRAPHICS_VERx100(xe) >= 2000) { 592 num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE); 593 xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1)); 594 595 ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1); 596 mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index); 597 598 } else { 599 num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size), 600 NUM_CCS_BYTES_PER_BLOCK); 601 xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1)); 602 603 ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1); 604 mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index); 605 } 606 607 *cs++ = XY_CTRL_SURF_COPY_BLT | 608 (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT | 609 (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT | 610 ccs_copy_size; 611 *cs++ = lower_32_bits(src_ofs); 612 *cs++ = upper_32_bits(src_ofs) | mocs; 613 *cs++ = lower_32_bits(dst_ofs); 614 *cs++ = upper_32_bits(dst_ofs) | mocs; 615 616 bb->len = cs - bb->cs; 617 } 618 619 #define EMIT_COPY_DW 10 620 static void emit_copy(struct xe_gt *gt, struct xe_bb *bb, 621 u64 src_ofs, u64 dst_ofs, unsigned int size, 622 unsigned int pitch) 623 { 624 struct xe_device *xe = gt_to_xe(gt); 625 u32 mocs = 0; 626 u32 tile_y = 0; 627 628 xe_gt_assert(gt, size / pitch <= S16_MAX); 629 xe_gt_assert(gt, pitch / 4 <= S16_MAX); 630 xe_gt_assert(gt, pitch <= U16_MAX); 631 632 if (GRAPHICS_VER(xe) >= 20) 633 mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index); 634 635 if (GRAPHICS_VERx100(xe) >= 1250) 636 tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4; 637 638 bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2); 639 bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs; 640 bb->cs[bb->len++] = 0; 641 bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; 642 bb->cs[bb->len++] = lower_32_bits(dst_ofs); 643 bb->cs[bb->len++] = upper_32_bits(dst_ofs); 644 bb->cs[bb->len++] = 0; 645 bb->cs[bb->len++] = pitch | mocs; 646 bb->cs[bb->len++] = lower_32_bits(src_ofs); 647 bb->cs[bb->len++] = upper_32_bits(src_ofs); 648 } 649 650 static int job_add_deps(struct xe_sched_job *job, struct dma_resv *resv, 651 enum dma_resv_usage usage) 652 { 653 return drm_sched_job_add_resv_dependencies(&job->drm, resv, usage); 654 } 655 656 static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm) 657 { 658 return usm ? m->usm_batch_base_ofs : m->batch_base_ofs; 659 } 660 661 static u32 xe_migrate_ccs_copy(struct xe_migrate *m, 662 struct xe_bb *bb, 663 u64 src_ofs, bool src_is_indirect, 664 u64 dst_ofs, bool dst_is_indirect, u32 dst_size, 665 u64 ccs_ofs, bool copy_ccs) 666 { 667 struct xe_gt *gt = m->tile->primary_gt; 668 u32 flush_flags = 0; 669 670 if (xe_device_has_flat_ccs(gt_to_xe(gt)) && !copy_ccs && dst_is_indirect) { 671 /* 672 * If the src is already in vram, then it should already 673 * have been cleared by us, or has been populated by the 674 * user. Make sure we copy the CCS aux state as-is. 675 * 676 * Otherwise if the bo doesn't have any CCS metadata attached, 677 * we still need to clear it for security reasons. 678 */ 679 u64 ccs_src_ofs = src_is_indirect ? src_ofs : m->cleared_mem_ofs; 680 681 emit_copy_ccs(gt, bb, 682 dst_ofs, true, 683 ccs_src_ofs, src_is_indirect, dst_size); 684 685 flush_flags = MI_FLUSH_DW_CCS; 686 } else if (copy_ccs) { 687 if (!src_is_indirect) 688 src_ofs = ccs_ofs; 689 else if (!dst_is_indirect) 690 dst_ofs = ccs_ofs; 691 692 xe_gt_assert(gt, src_is_indirect || dst_is_indirect); 693 694 emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs, 695 src_is_indirect, dst_size); 696 if (dst_is_indirect) 697 flush_flags = MI_FLUSH_DW_CCS; 698 } 699 700 return flush_flags; 701 } 702 703 /** 704 * xe_migrate_copy() - Copy content of TTM resources. 705 * @m: The migration context. 706 * @src_bo: The buffer object @src is currently bound to. 707 * @dst_bo: If copying between resources created for the same bo, set this to 708 * the same value as @src_bo. If copying between buffer objects, set it to 709 * the buffer object @dst is currently bound to. 710 * @src: The source TTM resource. 711 * @dst: The dst TTM resource. 712 * @copy_only_ccs: If true copy only CCS metadata 713 * 714 * Copies the contents of @src to @dst: On flat CCS devices, 715 * the CCS metadata is copied as well if needed, or if not present, 716 * the CCS metadata of @dst is cleared for security reasons. 717 * 718 * Return: Pointer to a dma_fence representing the last copy batch, or 719 * an error pointer on failure. If there is a failure, any copy operation 720 * started by the function call has been synced. 721 */ 722 struct dma_fence *xe_migrate_copy(struct xe_migrate *m, 723 struct xe_bo *src_bo, 724 struct xe_bo *dst_bo, 725 struct ttm_resource *src, 726 struct ttm_resource *dst, 727 bool copy_only_ccs) 728 { 729 struct xe_gt *gt = m->tile->primary_gt; 730 struct xe_device *xe = gt_to_xe(gt); 731 struct dma_fence *fence = NULL; 732 u64 size = src_bo->size; 733 struct xe_res_cursor src_it, dst_it, ccs_it; 734 u64 src_L0_ofs, dst_L0_ofs; 735 u32 src_L0_pt, dst_L0_pt; 736 u64 src_L0, dst_L0; 737 int pass = 0; 738 int err; 739 bool src_is_pltt = src->mem_type == XE_PL_TT; 740 bool dst_is_pltt = dst->mem_type == XE_PL_TT; 741 bool src_is_vram = mem_type_is_vram(src->mem_type); 742 bool dst_is_vram = mem_type_is_vram(dst->mem_type); 743 bool copy_ccs = xe_device_has_flat_ccs(xe) && 744 xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo); 745 bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram); 746 747 /* Copying CCS between two different BOs is not supported yet. */ 748 if (XE_WARN_ON(copy_ccs && src_bo != dst_bo)) 749 return ERR_PTR(-EINVAL); 750 751 if (src_bo != dst_bo && XE_WARN_ON(src_bo->size != dst_bo->size)) 752 return ERR_PTR(-EINVAL); 753 754 if (!src_is_vram) 755 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); 756 else 757 xe_res_first(src, 0, size, &src_it); 758 if (!dst_is_vram) 759 xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it); 760 else 761 xe_res_first(dst, 0, size, &dst_it); 762 763 if (copy_system_ccs) 764 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), 765 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), 766 &ccs_it); 767 768 while (size) { 769 u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */ 770 struct xe_sched_job *job; 771 struct xe_bb *bb; 772 u32 flush_flags; 773 u32 update_idx; 774 u64 ccs_ofs, ccs_size; 775 u32 ccs_pt; 776 777 bool usm = xe->info.has_usm; 778 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 779 780 src_L0 = xe_migrate_res_sizes(m, &src_it); 781 dst_L0 = xe_migrate_res_sizes(m, &dst_it); 782 783 drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n", 784 pass++, src_L0, dst_L0); 785 786 src_L0 = min(src_L0, dst_L0); 787 788 batch_size += pte_update_size(m, src_is_vram, src, &src_it, &src_L0, 789 &src_L0_ofs, &src_L0_pt, 0, 0, 790 avail_pts); 791 792 batch_size += pte_update_size(m, dst_is_vram, dst, &dst_it, &src_L0, 793 &dst_L0_ofs, &dst_L0_pt, 0, 794 avail_pts, avail_pts); 795 796 if (copy_system_ccs) { 797 ccs_size = xe_device_ccs_bytes(xe, src_L0); 798 batch_size += pte_update_size(m, false, NULL, &ccs_it, &ccs_size, 799 &ccs_ofs, &ccs_pt, 0, 800 2 * avail_pts, 801 avail_pts); 802 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); 803 } 804 805 /* Add copy commands size here */ 806 batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + 807 ((xe_device_has_flat_ccs(xe) ? EMIT_COPY_CCS_DW : 0)); 808 809 bb = xe_bb_new(gt, batch_size, usm); 810 if (IS_ERR(bb)) { 811 err = PTR_ERR(bb); 812 goto err_sync; 813 } 814 815 if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it)) 816 xe_res_next(&src_it, src_L0); 817 else 818 emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs, 819 &src_it, src_L0, src); 820 821 if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it)) 822 xe_res_next(&dst_it, src_L0); 823 else 824 emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs, 825 &dst_it, src_L0, dst); 826 827 if (copy_system_ccs) 828 emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); 829 830 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 831 update_idx = bb->len; 832 833 if (!copy_only_ccs) 834 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE); 835 836 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, 837 IS_DGFX(xe) ? src_is_vram : src_is_pltt, 838 dst_L0_ofs, 839 IS_DGFX(xe) ? dst_is_vram : dst_is_pltt, 840 src_L0, ccs_ofs, copy_ccs); 841 842 job = xe_bb_create_migration_job(m->q, bb, 843 xe_migrate_batch_base(m, usm), 844 update_idx); 845 if (IS_ERR(job)) { 846 err = PTR_ERR(job); 847 goto err; 848 } 849 850 xe_sched_job_add_migrate_flush(job, flush_flags); 851 if (!fence) { 852 err = job_add_deps(job, src_bo->ttm.base.resv, 853 DMA_RESV_USAGE_BOOKKEEP); 854 if (!err && src_bo != dst_bo) 855 err = job_add_deps(job, dst_bo->ttm.base.resv, 856 DMA_RESV_USAGE_BOOKKEEP); 857 if (err) 858 goto err_job; 859 } 860 861 mutex_lock(&m->job_mutex); 862 xe_sched_job_arm(job); 863 dma_fence_put(fence); 864 fence = dma_fence_get(&job->drm.s_fence->finished); 865 xe_sched_job_push(job); 866 867 dma_fence_put(m->fence); 868 m->fence = dma_fence_get(fence); 869 870 mutex_unlock(&m->job_mutex); 871 872 xe_bb_free(bb, fence); 873 size -= src_L0; 874 continue; 875 876 err_job: 877 xe_sched_job_put(job); 878 err: 879 xe_bb_free(bb, NULL); 880 881 err_sync: 882 /* Sync partial copy if any. FIXME: under job_mutex? */ 883 if (fence) { 884 dma_fence_wait(fence, false); 885 dma_fence_put(fence); 886 } 887 888 return ERR_PTR(err); 889 } 890 891 return fence; 892 } 893 894 static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 895 u32 size, u32 pitch) 896 { 897 struct xe_device *xe = gt_to_xe(gt); 898 u32 *cs = bb->cs + bb->len; 899 u32 len = PVC_MEM_SET_CMD_LEN_DW; 900 901 *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2); 902 *cs++ = pitch - 1; 903 *cs++ = (size / pitch) - 1; 904 *cs++ = pitch - 1; 905 *cs++ = lower_32_bits(src_ofs); 906 *cs++ = upper_32_bits(src_ofs); 907 if (GRAPHICS_VERx100(xe) >= 2000) 908 *cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 909 else 910 *cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 911 912 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 913 914 bb->len += len; 915 } 916 917 static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, 918 u64 src_ofs, u32 size, u32 pitch, bool is_vram) 919 { 920 struct xe_device *xe = gt_to_xe(gt); 921 u32 *cs = bb->cs + bb->len; 922 u32 len = XY_FAST_COLOR_BLT_DW; 923 924 if (GRAPHICS_VERx100(xe) < 1250) 925 len = 11; 926 927 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | 928 (len - 2); 929 if (GRAPHICS_VERx100(xe) >= 2000) 930 *cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) | 931 (pitch - 1); 932 else 933 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) | 934 (pitch - 1); 935 *cs++ = 0; 936 *cs++ = (size / pitch) << 16 | pitch / 4; 937 *cs++ = lower_32_bits(src_ofs); 938 *cs++ = upper_32_bits(src_ofs); 939 *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; 940 *cs++ = 0; 941 *cs++ = 0; 942 *cs++ = 0; 943 *cs++ = 0; 944 945 if (len > 11) { 946 *cs++ = 0; 947 *cs++ = 0; 948 *cs++ = 0; 949 *cs++ = 0; 950 *cs++ = 0; 951 } 952 953 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 954 955 bb->len += len; 956 } 957 958 static bool has_service_copy_support(struct xe_gt *gt) 959 { 960 /* 961 * What we care about is whether the architecture was designed with 962 * service copy functionality (specifically the new MEM_SET / MEM_COPY 963 * instructions) so check the architectural engine list rather than the 964 * actual list since these instructions are usable on BCS0 even if 965 * all of the actual service copy engines (BCS1-BCS8) have been fused 966 * off. 967 */ 968 return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8, 969 XE_HW_ENGINE_BCS1); 970 } 971 972 static u32 emit_clear_cmd_len(struct xe_gt *gt) 973 { 974 if (has_service_copy_support(gt)) 975 return PVC_MEM_SET_CMD_LEN_DW; 976 else 977 return XY_FAST_COLOR_BLT_DW; 978 } 979 980 static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 981 u32 size, u32 pitch, bool is_vram) 982 { 983 if (has_service_copy_support(gt)) 984 emit_clear_link_copy(gt, bb, src_ofs, size, pitch); 985 else 986 emit_clear_main_copy(gt, bb, src_ofs, size, pitch, 987 is_vram); 988 } 989 990 /** 991 * xe_migrate_clear() - Copy content of TTM resources. 992 * @m: The migration context. 993 * @bo: The buffer object @dst is currently bound to. 994 * @dst: The dst TTM resource to be cleared. 995 * 996 * Clear the contents of @dst to zero. On flat CCS devices, 997 * the CCS metadata is cleared to zero as well on VRAM destinations. 998 * TODO: Eliminate the @bo argument. 999 * 1000 * Return: Pointer to a dma_fence representing the last clear batch, or 1001 * an error pointer on failure. If there is a failure, any clear operation 1002 * started by the function call has been synced. 1003 */ 1004 struct dma_fence *xe_migrate_clear(struct xe_migrate *m, 1005 struct xe_bo *bo, 1006 struct ttm_resource *dst) 1007 { 1008 bool clear_vram = mem_type_is_vram(dst->mem_type); 1009 struct xe_gt *gt = m->tile->primary_gt; 1010 struct xe_device *xe = gt_to_xe(gt); 1011 bool clear_system_ccs = (xe_bo_needs_ccs_pages(bo) && !IS_DGFX(xe)) ? true : false; 1012 struct dma_fence *fence = NULL; 1013 u64 size = bo->size; 1014 struct xe_res_cursor src_it; 1015 struct ttm_resource *src = dst; 1016 int err; 1017 1018 if (!clear_vram) 1019 xe_res_first_sg(xe_bo_sg(bo), 0, bo->size, &src_it); 1020 else 1021 xe_res_first(src, 0, bo->size, &src_it); 1022 1023 while (size) { 1024 u64 clear_L0_ofs; 1025 u32 clear_L0_pt; 1026 u32 flush_flags = 0; 1027 u64 clear_L0; 1028 struct xe_sched_job *job; 1029 struct xe_bb *bb; 1030 u32 batch_size, update_idx; 1031 1032 bool usm = xe->info.has_usm; 1033 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 1034 1035 clear_L0 = xe_migrate_res_sizes(m, &src_it); 1036 1037 /* Calculate final sizes and batch size.. */ 1038 batch_size = 2 + 1039 pte_update_size(m, clear_vram, src, &src_it, 1040 &clear_L0, &clear_L0_ofs, &clear_L0_pt, 1041 clear_system_ccs ? 0 : emit_clear_cmd_len(gt), 0, 1042 avail_pts); 1043 1044 if (xe_device_has_flat_ccs(xe)) 1045 batch_size += EMIT_COPY_CCS_DW; 1046 1047 /* Clear commands */ 1048 1049 if (WARN_ON_ONCE(!clear_L0)) 1050 break; 1051 1052 bb = xe_bb_new(gt, batch_size, usm); 1053 if (IS_ERR(bb)) { 1054 err = PTR_ERR(bb); 1055 goto err_sync; 1056 } 1057 1058 size -= clear_L0; 1059 /* Preemption is enabled again by the ring ops. */ 1060 if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it)) 1061 xe_res_next(&src_it, clear_L0); 1062 else 1063 emit_pte(m, bb, clear_L0_pt, clear_vram, clear_system_ccs, 1064 &src_it, clear_L0, dst); 1065 1066 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1067 update_idx = bb->len; 1068 1069 if (!clear_system_ccs) 1070 emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram); 1071 1072 if (xe_device_has_flat_ccs(xe)) { 1073 emit_copy_ccs(gt, bb, clear_L0_ofs, true, 1074 m->cleared_mem_ofs, false, clear_L0); 1075 flush_flags = MI_FLUSH_DW_CCS; 1076 } 1077 1078 job = xe_bb_create_migration_job(m->q, bb, 1079 xe_migrate_batch_base(m, usm), 1080 update_idx); 1081 if (IS_ERR(job)) { 1082 err = PTR_ERR(job); 1083 goto err; 1084 } 1085 1086 xe_sched_job_add_migrate_flush(job, flush_flags); 1087 if (!fence) { 1088 /* 1089 * There can't be anything userspace related at this 1090 * point, so we just need to respect any potential move 1091 * fences, which are always tracked as 1092 * DMA_RESV_USAGE_KERNEL. 1093 */ 1094 err = job_add_deps(job, bo->ttm.base.resv, 1095 DMA_RESV_USAGE_KERNEL); 1096 if (err) 1097 goto err_job; 1098 } 1099 1100 mutex_lock(&m->job_mutex); 1101 xe_sched_job_arm(job); 1102 dma_fence_put(fence); 1103 fence = dma_fence_get(&job->drm.s_fence->finished); 1104 xe_sched_job_push(job); 1105 1106 dma_fence_put(m->fence); 1107 m->fence = dma_fence_get(fence); 1108 1109 mutex_unlock(&m->job_mutex); 1110 1111 xe_bb_free(bb, fence); 1112 continue; 1113 1114 err_job: 1115 xe_sched_job_put(job); 1116 err: 1117 xe_bb_free(bb, NULL); 1118 err_sync: 1119 /* Sync partial copies if any. FIXME: job_mutex? */ 1120 if (fence) { 1121 dma_fence_wait(m->fence, false); 1122 dma_fence_put(fence); 1123 } 1124 1125 return ERR_PTR(err); 1126 } 1127 1128 if (clear_system_ccs) 1129 bo->ccs_cleared = true; 1130 1131 return fence; 1132 } 1133 1134 static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs, 1135 const struct xe_vm_pgtable_update *update, 1136 struct xe_migrate_pt_update *pt_update) 1137 { 1138 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1139 u32 chunk; 1140 u32 ofs = update->ofs, size = update->qwords; 1141 1142 /* 1143 * If we have 512 entries (max), we would populate it ourselves, 1144 * and update the PDE above it to the new pointer. 1145 * The only time this can only happen if we have to update the top 1146 * PDE. This requires a BO that is almost vm->size big. 1147 * 1148 * This shouldn't be possible in practice.. might change when 16K 1149 * pages are used. Hence the assert. 1150 */ 1151 xe_tile_assert(tile, update->qwords < MAX_NUM_PTE); 1152 if (!ppgtt_ofs) 1153 ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile), 1154 xe_bo_addr(update->pt_bo, 0, 1155 XE_PAGE_SIZE)); 1156 1157 do { 1158 u64 addr = ppgtt_ofs + ofs * 8; 1159 1160 chunk = min(size, MAX_PTE_PER_SDI); 1161 1162 /* Ensure populatefn can do memset64 by aligning bb->cs */ 1163 if (!(bb->len & 1)) 1164 bb->cs[bb->len++] = MI_NOOP; 1165 1166 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 1167 bb->cs[bb->len++] = lower_32_bits(addr); 1168 bb->cs[bb->len++] = upper_32_bits(addr); 1169 ops->populate(pt_update, tile, NULL, bb->cs + bb->len, ofs, chunk, 1170 update); 1171 1172 bb->len += chunk * 2; 1173 ofs += chunk; 1174 size -= chunk; 1175 } while (size); 1176 } 1177 1178 struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) 1179 { 1180 return xe_vm_get(m->q->vm); 1181 } 1182 1183 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1184 struct migrate_test_params { 1185 struct xe_test_priv base; 1186 bool force_gpu; 1187 }; 1188 1189 #define to_migrate_test_params(_priv) \ 1190 container_of(_priv, struct migrate_test_params, base) 1191 #endif 1192 1193 static struct dma_fence * 1194 xe_migrate_update_pgtables_cpu(struct xe_migrate *m, 1195 struct xe_vm *vm, struct xe_bo *bo, 1196 const struct xe_vm_pgtable_update *updates, 1197 u32 num_updates, bool wait_vm, 1198 struct xe_migrate_pt_update *pt_update) 1199 { 1200 XE_TEST_DECLARE(struct migrate_test_params *test = 1201 to_migrate_test_params 1202 (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));) 1203 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1204 struct dma_fence *fence; 1205 int err; 1206 u32 i; 1207 1208 if (XE_TEST_ONLY(test && test->force_gpu)) 1209 return ERR_PTR(-ETIME); 1210 1211 if (bo && !dma_resv_test_signaled(bo->ttm.base.resv, 1212 DMA_RESV_USAGE_KERNEL)) 1213 return ERR_PTR(-ETIME); 1214 1215 if (wait_vm && !dma_resv_test_signaled(xe_vm_resv(vm), 1216 DMA_RESV_USAGE_BOOKKEEP)) 1217 return ERR_PTR(-ETIME); 1218 1219 if (ops->pre_commit) { 1220 pt_update->job = NULL; 1221 err = ops->pre_commit(pt_update); 1222 if (err) 1223 return ERR_PTR(err); 1224 } 1225 for (i = 0; i < num_updates; i++) { 1226 const struct xe_vm_pgtable_update *update = &updates[i]; 1227 1228 ops->populate(pt_update, m->tile, &update->pt_bo->vmap, NULL, 1229 update->ofs, update->qwords, update); 1230 } 1231 1232 if (vm) { 1233 trace_xe_vm_cpu_bind(vm); 1234 xe_device_wmb(vm->xe); 1235 } 1236 1237 fence = dma_fence_get_stub(); 1238 1239 return fence; 1240 } 1241 1242 static bool no_in_syncs(struct xe_vm *vm, struct xe_exec_queue *q, 1243 struct xe_sync_entry *syncs, u32 num_syncs) 1244 { 1245 struct dma_fence *fence; 1246 int i; 1247 1248 for (i = 0; i < num_syncs; i++) { 1249 fence = syncs[i].fence; 1250 1251 if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, 1252 &fence->flags)) 1253 return false; 1254 } 1255 if (q) { 1256 fence = xe_exec_queue_last_fence_get(q, vm); 1257 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)) { 1258 dma_fence_put(fence); 1259 return false; 1260 } 1261 dma_fence_put(fence); 1262 } 1263 1264 return true; 1265 } 1266 1267 /** 1268 * xe_migrate_update_pgtables() - Pipelined page-table update 1269 * @m: The migrate context. 1270 * @vm: The vm we'll be updating. 1271 * @bo: The bo whose dma-resv we will await before updating, or NULL if userptr. 1272 * @q: The exec queue to be used for the update or NULL if the default 1273 * migration engine is to be used. 1274 * @updates: An array of update descriptors. 1275 * @num_updates: Number of descriptors in @updates. 1276 * @syncs: Array of xe_sync_entry to await before updating. Note that waits 1277 * will block the engine timeline. 1278 * @num_syncs: Number of entries in @syncs. 1279 * @pt_update: Pointer to a struct xe_migrate_pt_update, which contains 1280 * pointers to callback functions and, if subclassed, private arguments to 1281 * those. 1282 * 1283 * Perform a pipelined page-table update. The update descriptors are typically 1284 * built under the same lock critical section as a call to this function. If 1285 * using the default engine for the updates, they will be performed in the 1286 * order they grab the job_mutex. If different engines are used, external 1287 * synchronization is needed for overlapping updates to maintain page-table 1288 * consistency. Note that the meaing of "overlapping" is that the updates 1289 * touch the same page-table, which might be a higher-level page-directory. 1290 * If no pipelining is needed, then updates may be performed by the cpu. 1291 * 1292 * Return: A dma_fence that, when signaled, indicates the update completion. 1293 */ 1294 struct dma_fence * 1295 xe_migrate_update_pgtables(struct xe_migrate *m, 1296 struct xe_vm *vm, 1297 struct xe_bo *bo, 1298 struct xe_exec_queue *q, 1299 const struct xe_vm_pgtable_update *updates, 1300 u32 num_updates, 1301 struct xe_sync_entry *syncs, u32 num_syncs, 1302 struct xe_migrate_pt_update *pt_update) 1303 { 1304 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1305 struct xe_tile *tile = m->tile; 1306 struct xe_gt *gt = tile->primary_gt; 1307 struct xe_device *xe = tile_to_xe(tile); 1308 struct xe_sched_job *job; 1309 struct dma_fence *fence; 1310 struct drm_suballoc *sa_bo = NULL; 1311 struct xe_vma *vma = pt_update->vma; 1312 struct xe_bb *bb; 1313 u32 i, batch_size, ppgtt_ofs, update_idx, page_ofs = 0; 1314 u64 addr; 1315 int err = 0; 1316 bool usm = !q && xe->info.has_usm; 1317 bool first_munmap_rebind = vma && 1318 vma->gpuva.flags & XE_VMA_FIRST_REBIND; 1319 struct xe_exec_queue *q_override = !q ? m->q : q; 1320 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 1321 1322 /* Use the CPU if no in syncs and engine is idle */ 1323 if (no_in_syncs(vm, q, syncs, num_syncs) && xe_exec_queue_is_idle(q_override)) { 1324 fence = xe_migrate_update_pgtables_cpu(m, vm, bo, updates, 1325 num_updates, 1326 first_munmap_rebind, 1327 pt_update); 1328 if (!IS_ERR(fence) || fence == ERR_PTR(-EAGAIN)) 1329 return fence; 1330 } 1331 1332 /* fixed + PTE entries */ 1333 if (IS_DGFX(xe)) 1334 batch_size = 2; 1335 else 1336 batch_size = 6 + num_updates * 2; 1337 1338 for (i = 0; i < num_updates; i++) { 1339 u32 num_cmds = DIV_ROUND_UP(updates[i].qwords, MAX_PTE_PER_SDI); 1340 1341 /* align noop + MI_STORE_DATA_IMM cmd prefix */ 1342 batch_size += 4 * num_cmds + updates[i].qwords * 2; 1343 } 1344 1345 /* 1346 * XXX: Create temp bo to copy from, if batch_size becomes too big? 1347 * 1348 * Worst case: Sum(2 * (each lower level page size) + (top level page size)) 1349 * Should be reasonably bound.. 1350 */ 1351 xe_tile_assert(tile, batch_size < SZ_128K); 1352 1353 bb = xe_bb_new(gt, batch_size, !q && xe->info.has_usm); 1354 if (IS_ERR(bb)) 1355 return ERR_CAST(bb); 1356 1357 /* For sysmem PTE's, need to map them in our hole.. */ 1358 if (!IS_DGFX(xe)) { 1359 ppgtt_ofs = NUM_KERNEL_PDE - 1; 1360 if (q) { 1361 xe_tile_assert(tile, num_updates <= NUM_VMUSA_WRITES_PER_UNIT); 1362 1363 sa_bo = drm_suballoc_new(&m->vm_update_sa, 1, 1364 GFP_KERNEL, true, 0); 1365 if (IS_ERR(sa_bo)) { 1366 err = PTR_ERR(sa_bo); 1367 goto err; 1368 } 1369 1370 ppgtt_ofs = NUM_KERNEL_PDE + 1371 (drm_suballoc_soffset(sa_bo) / 1372 NUM_VMUSA_UNIT_PER_PAGE); 1373 page_ofs = (drm_suballoc_soffset(sa_bo) % 1374 NUM_VMUSA_UNIT_PER_PAGE) * 1375 VM_SA_UPDATE_UNIT_SIZE; 1376 } 1377 1378 /* Map our PT's to gtt */ 1379 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(num_updates); 1380 bb->cs[bb->len++] = ppgtt_ofs * XE_PAGE_SIZE + page_ofs; 1381 bb->cs[bb->len++] = 0; /* upper_32_bits */ 1382 1383 for (i = 0; i < num_updates; i++) { 1384 struct xe_bo *pt_bo = updates[i].pt_bo; 1385 1386 xe_tile_assert(tile, pt_bo->size == SZ_4K); 1387 1388 addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, pat_index, 0); 1389 bb->cs[bb->len++] = lower_32_bits(addr); 1390 bb->cs[bb->len++] = upper_32_bits(addr); 1391 } 1392 1393 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1394 update_idx = bb->len; 1395 1396 addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + 1397 (page_ofs / sizeof(u64)) * XE_PAGE_SIZE; 1398 for (i = 0; i < num_updates; i++) 1399 write_pgtable(tile, bb, addr + i * XE_PAGE_SIZE, 1400 &updates[i], pt_update); 1401 } else { 1402 /* phys pages, no preamble required */ 1403 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1404 update_idx = bb->len; 1405 1406 for (i = 0; i < num_updates; i++) 1407 write_pgtable(tile, bb, 0, &updates[i], pt_update); 1408 } 1409 1410 job = xe_bb_create_migration_job(q ?: m->q, bb, 1411 xe_migrate_batch_base(m, usm), 1412 update_idx); 1413 if (IS_ERR(job)) { 1414 err = PTR_ERR(job); 1415 goto err_bb; 1416 } 1417 1418 /* Wait on BO move */ 1419 if (bo) { 1420 err = job_add_deps(job, bo->ttm.base.resv, 1421 DMA_RESV_USAGE_KERNEL); 1422 if (err) 1423 goto err_job; 1424 } 1425 1426 /* 1427 * Munmap style VM unbind, need to wait for all jobs to be complete / 1428 * trigger preempts before moving forward 1429 */ 1430 if (first_munmap_rebind) { 1431 err = job_add_deps(job, xe_vm_resv(vm), 1432 DMA_RESV_USAGE_BOOKKEEP); 1433 if (err) 1434 goto err_job; 1435 } 1436 1437 err = xe_sched_job_last_fence_add_dep(job, vm); 1438 for (i = 0; !err && i < num_syncs; i++) 1439 err = xe_sync_entry_add_deps(&syncs[i], job); 1440 1441 if (err) 1442 goto err_job; 1443 1444 if (ops->pre_commit) { 1445 pt_update->job = job; 1446 err = ops->pre_commit(pt_update); 1447 if (err) 1448 goto err_job; 1449 } 1450 if (!q) 1451 mutex_lock(&m->job_mutex); 1452 1453 xe_sched_job_arm(job); 1454 fence = dma_fence_get(&job->drm.s_fence->finished); 1455 xe_sched_job_push(job); 1456 1457 if (!q) 1458 mutex_unlock(&m->job_mutex); 1459 1460 xe_bb_free(bb, fence); 1461 drm_suballoc_free(sa_bo, fence); 1462 1463 return fence; 1464 1465 err_job: 1466 xe_sched_job_put(job); 1467 err_bb: 1468 xe_bb_free(bb, NULL); 1469 err: 1470 drm_suballoc_free(sa_bo, NULL); 1471 return ERR_PTR(err); 1472 } 1473 1474 /** 1475 * xe_migrate_wait() - Complete all operations using the xe_migrate context 1476 * @m: Migrate context to wait for. 1477 * 1478 * Waits until the GPU no longer uses the migrate context's default engine 1479 * or its page-table objects. FIXME: What about separate page-table update 1480 * engines? 1481 */ 1482 void xe_migrate_wait(struct xe_migrate *m) 1483 { 1484 if (m->fence) 1485 dma_fence_wait(m->fence, false); 1486 } 1487 1488 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1489 #include "tests/xe_migrate.c" 1490 #endif 1491