1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2020 Intel Corporation 4 */ 5 6 #include "xe_migrate.h" 7 8 #include <linux/bitfield.h> 9 #include <linux/sizes.h> 10 11 #include <drm/drm_managed.h> 12 #include <drm/ttm/ttm_tt.h> 13 #include <uapi/drm/xe_drm.h> 14 15 #include <generated/xe_wa_oob.h> 16 17 #include "instructions/xe_gpu_commands.h" 18 #include "instructions/xe_mi_commands.h" 19 #include "regs/xe_gtt_defs.h" 20 #include "tests/xe_test.h" 21 #include "xe_assert.h" 22 #include "xe_bb.h" 23 #include "xe_bo.h" 24 #include "xe_exec_queue.h" 25 #include "xe_ggtt.h" 26 #include "xe_gt.h" 27 #include "xe_hw_engine.h" 28 #include "xe_lrc.h" 29 #include "xe_map.h" 30 #include "xe_mocs.h" 31 #include "xe_pt.h" 32 #include "xe_res_cursor.h" 33 #include "xe_sched_job.h" 34 #include "xe_sync.h" 35 #include "xe_trace_bo.h" 36 #include "xe_vm.h" 37 38 /** 39 * struct xe_migrate - migrate context. 40 */ 41 struct xe_migrate { 42 /** @q: Default exec queue used for migration */ 43 struct xe_exec_queue *q; 44 /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */ 45 struct xe_tile *tile; 46 /** @job_mutex: Timeline mutex for @eng. */ 47 struct mutex job_mutex; 48 /** @pt_bo: Page-table buffer object. */ 49 struct xe_bo *pt_bo; 50 /** @batch_base_ofs: VM offset of the migration batch buffer */ 51 u64 batch_base_ofs; 52 /** @usm_batch_base_ofs: VM offset of the usm batch buffer */ 53 u64 usm_batch_base_ofs; 54 /** @cleared_mem_ofs: VM offset of @cleared_bo. */ 55 u64 cleared_mem_ofs; 56 /** 57 * @fence: dma-fence representing the last migration job batch. 58 * Protected by @job_mutex. 59 */ 60 struct dma_fence *fence; 61 /** 62 * @vm_update_sa: For integrated, used to suballocate page-tables 63 * out of the pt_bo. 64 */ 65 struct drm_suballoc_manager vm_update_sa; 66 /** @min_chunk_size: For dgfx, Minimum chunk size */ 67 u64 min_chunk_size; 68 }; 69 70 #define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ 71 #define MAX_CCS_LIMITED_TRANSFER SZ_4M /* XE_PAGE_SIZE * (FIELD_MAX(XE2_CCS_SIZE_MASK) + 1) */ 72 #define NUM_KERNEL_PDE 15 73 #define NUM_PT_SLOTS 32 74 #define LEVEL0_PAGE_TABLE_ENCODE_SIZE SZ_2M 75 #define MAX_NUM_PTE 512 76 #define IDENTITY_OFFSET 256ULL 77 78 /* 79 * Although MI_STORE_DATA_IMM's "length" field is 10-bits, 0x3FE is the largest 80 * legal value accepted. Since that instruction field is always stored in 81 * (val-2) format, this translates to 0x400 dwords for the true maximum length 82 * of the instruction. Subtracting the instruction header (1 dword) and 83 * address (2 dwords), that leaves 0x3FD dwords (0x1FE qwords) for PTE values. 84 */ 85 #define MAX_PTE_PER_SDI 0x1FE 86 87 /** 88 * xe_tile_migrate_exec_queue() - Get this tile's migrate exec queue. 89 * @tile: The tile. 90 * 91 * Returns the default migrate exec queue of this tile. 92 * 93 * Return: The default migrate exec queue 94 */ 95 struct xe_exec_queue *xe_tile_migrate_exec_queue(struct xe_tile *tile) 96 { 97 return tile->migrate->q; 98 } 99 100 static void xe_migrate_fini(struct drm_device *dev, void *arg) 101 { 102 struct xe_migrate *m = arg; 103 104 xe_vm_lock(m->q->vm, false); 105 xe_bo_unpin(m->pt_bo); 106 xe_vm_unlock(m->q->vm); 107 108 dma_fence_put(m->fence); 109 xe_bo_put(m->pt_bo); 110 drm_suballoc_manager_fini(&m->vm_update_sa); 111 mutex_destroy(&m->job_mutex); 112 xe_vm_close_and_put(m->q->vm); 113 xe_exec_queue_put(m->q); 114 } 115 116 static u64 xe_migrate_vm_addr(u64 slot, u32 level) 117 { 118 XE_WARN_ON(slot >= NUM_PT_SLOTS); 119 120 /* First slot is reserved for mapping of PT bo and bb, start from 1 */ 121 return (slot + 1ULL) << xe_pt_shift(level + 1); 122 } 123 124 static u64 xe_migrate_vram_ofs(struct xe_device *xe, u64 addr, bool is_comp_pte) 125 { 126 /* 127 * Remove the DPA to get a correct offset into identity table for the 128 * migrate offset 129 */ 130 u64 identity_offset = IDENTITY_OFFSET; 131 132 if (GRAPHICS_VER(xe) >= 20 && is_comp_pte) 133 identity_offset += DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G); 134 135 addr -= xe->mem.vram.dpa_base; 136 return addr + (identity_offset << xe_pt_shift(2)); 137 } 138 139 static void xe_migrate_program_identity(struct xe_device *xe, struct xe_vm *vm, struct xe_bo *bo, 140 u64 map_ofs, u64 vram_offset, u16 pat_index, u64 pt_2m_ofs) 141 { 142 u64 pos, ofs, flags; 143 u64 entry; 144 /* XXX: Unclear if this should be usable_size? */ 145 u64 vram_limit = xe->mem.vram.actual_physical_size + 146 xe->mem.vram.dpa_base; 147 u32 level = 2; 148 149 ofs = map_ofs + XE_PAGE_SIZE * level + vram_offset * 8; 150 flags = vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, 151 true, 0); 152 153 xe_assert(xe, IS_ALIGNED(xe->mem.vram.usable_size, SZ_2M)); 154 155 /* 156 * Use 1GB pages when possible, last chunk always use 2M 157 * pages as mixing reserved memory (stolen, WOCPM) with a single 158 * mapping is not allowed on certain platforms. 159 */ 160 for (pos = xe->mem.vram.dpa_base; pos < vram_limit; 161 pos += SZ_1G, ofs += 8) { 162 if (pos + SZ_1G >= vram_limit) { 163 entry = vm->pt_ops->pde_encode_bo(bo, pt_2m_ofs, 164 pat_index); 165 xe_map_wr(xe, &bo->vmap, ofs, u64, entry); 166 167 flags = vm->pt_ops->pte_encode_addr(xe, 0, 168 pat_index, 169 level - 1, 170 true, 0); 171 172 for (ofs = pt_2m_ofs; pos < vram_limit; 173 pos += SZ_2M, ofs += 8) 174 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 175 break; /* Ensure pos == vram_limit assert correct */ 176 } 177 178 xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); 179 } 180 181 xe_assert(xe, pos == vram_limit); 182 } 183 184 static int xe_migrate_prepare_vm(struct xe_tile *tile, struct xe_migrate *m, 185 struct xe_vm *vm) 186 { 187 struct xe_device *xe = tile_to_xe(tile); 188 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 189 u8 id = tile->id; 190 u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; 191 #define VRAM_IDENTITY_MAP_COUNT 2 192 u32 num_setup = num_level + VRAM_IDENTITY_MAP_COUNT; 193 #undef VRAM_IDENTITY_MAP_COUNT 194 u32 map_ofs, level, i; 195 struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo; 196 u64 entry, pt29_ofs; 197 198 /* Can't bump NUM_PT_SLOTS too high */ 199 BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/XE_PAGE_SIZE); 200 /* Must be a multiple of 64K to support all platforms */ 201 BUILD_BUG_ON(NUM_PT_SLOTS * XE_PAGE_SIZE % SZ_64K); 202 /* And one slot reserved for the 4KiB page table updates */ 203 BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1)); 204 205 /* Need to be sure everything fits in the first PT, or create more */ 206 xe_tile_assert(tile, m->batch_base_ofs + batch->size < SZ_2M); 207 208 bo = xe_bo_create_pin_map(vm->xe, tile, vm, 209 num_entries * XE_PAGE_SIZE, 210 ttm_bo_type_kernel, 211 XE_BO_FLAG_VRAM_IF_DGFX(tile) | 212 XE_BO_FLAG_PINNED | 213 XE_BO_FLAG_PAGETABLE); 214 if (IS_ERR(bo)) 215 return PTR_ERR(bo); 216 217 /* PT30 & PT31 reserved for 2M identity map */ 218 pt29_ofs = bo->size - 3 * XE_PAGE_SIZE; 219 entry = vm->pt_ops->pde_encode_bo(bo, pt29_ofs, pat_index); 220 xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); 221 222 map_ofs = (num_entries - num_setup) * XE_PAGE_SIZE; 223 224 /* Map the entire BO in our level 0 pt */ 225 for (i = 0, level = 0; i < num_entries; level++) { 226 entry = vm->pt_ops->pte_encode_bo(bo, i * XE_PAGE_SIZE, 227 pat_index, 0); 228 229 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry); 230 231 if (vm->flags & XE_VM_FLAG_64K) 232 i += 16; 233 else 234 i += 1; 235 } 236 237 if (!IS_DGFX(xe)) { 238 /* Write out batch too */ 239 m->batch_base_ofs = NUM_PT_SLOTS * XE_PAGE_SIZE; 240 for (i = 0; i < batch->size; 241 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 242 XE_PAGE_SIZE) { 243 entry = vm->pt_ops->pte_encode_bo(batch, i, 244 pat_index, 0); 245 246 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 247 entry); 248 level++; 249 } 250 if (xe->info.has_usm) { 251 xe_tile_assert(tile, batch->size == SZ_1M); 252 253 batch = tile->primary_gt->usm.bb_pool->bo; 254 m->usm_batch_base_ofs = m->batch_base_ofs + SZ_1M; 255 xe_tile_assert(tile, batch->size == SZ_512K); 256 257 for (i = 0; i < batch->size; 258 i += vm->flags & XE_VM_FLAG_64K ? XE_64K_PAGE_SIZE : 259 XE_PAGE_SIZE) { 260 entry = vm->pt_ops->pte_encode_bo(batch, i, 261 pat_index, 0); 262 263 xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, 264 entry); 265 level++; 266 } 267 } 268 } else { 269 u64 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 270 271 m->batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false); 272 273 if (xe->info.has_usm) { 274 batch = tile->primary_gt->usm.bb_pool->bo; 275 batch_addr = xe_bo_addr(batch, 0, XE_PAGE_SIZE); 276 m->usm_batch_base_ofs = xe_migrate_vram_ofs(xe, batch_addr, false); 277 } 278 } 279 280 for (level = 1; level < num_level; level++) { 281 u32 flags = 0; 282 283 if (vm->flags & XE_VM_FLAG_64K && level == 1) 284 flags = XE_PDE_64K; 285 286 entry = vm->pt_ops->pde_encode_bo(bo, map_ofs + (u64)(level - 1) * 287 XE_PAGE_SIZE, pat_index); 288 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level, u64, 289 entry | flags); 290 } 291 292 /* Write PDE's that point to our BO. */ 293 for (i = 0; i < map_ofs / PAGE_SIZE; i++) { 294 entry = vm->pt_ops->pde_encode_bo(bo, (u64)i * XE_PAGE_SIZE, 295 pat_index); 296 297 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE + 298 (i + 1) * 8, u64, entry); 299 } 300 301 /* Set up a 1GiB NULL mapping at 255GiB offset. */ 302 level = 2; 303 xe_map_wr(xe, &bo->vmap, map_ofs + XE_PAGE_SIZE * level + 255 * 8, u64, 304 vm->pt_ops->pte_encode_addr(xe, 0, pat_index, level, IS_DGFX(xe), 0) 305 | XE_PTE_NULL); 306 m->cleared_mem_ofs = (255ULL << xe_pt_shift(level)); 307 308 /* Identity map the entire vram at 256GiB offset */ 309 if (IS_DGFX(xe)) { 310 u64 pt30_ofs = bo->size - 2 * XE_PAGE_SIZE; 311 312 xe_migrate_program_identity(xe, vm, bo, map_ofs, IDENTITY_OFFSET, 313 pat_index, pt30_ofs); 314 xe_assert(xe, xe->mem.vram.actual_physical_size <= 315 (MAX_NUM_PTE - IDENTITY_OFFSET) * SZ_1G); 316 317 /* 318 * Identity map the entire vram for compressed pat_index for xe2+ 319 * if flat ccs is enabled. 320 */ 321 if (GRAPHICS_VER(xe) >= 20 && xe_device_has_flat_ccs(xe)) { 322 u16 comp_pat_index = xe->pat.idx[XE_CACHE_NONE_COMPRESSION]; 323 u64 vram_offset = IDENTITY_OFFSET + 324 DIV_ROUND_UP_ULL(xe->mem.vram.actual_physical_size, SZ_1G); 325 u64 pt31_ofs = bo->size - XE_PAGE_SIZE; 326 327 xe_assert(xe, xe->mem.vram.actual_physical_size <= (MAX_NUM_PTE - 328 IDENTITY_OFFSET - IDENTITY_OFFSET / 2) * SZ_1G); 329 xe_migrate_program_identity(xe, vm, bo, map_ofs, vram_offset, 330 comp_pat_index, pt31_ofs); 331 } 332 } 333 334 /* 335 * Example layout created above, with root level = 3: 336 * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's 337 * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's 338 * [PT9...PT26]: Userspace PT's for VM_BIND, 4 KiB PTE's 339 * [PT27 = PDE 0] [PT28 = PDE 1] [PT29 = PDE 2] [PT30 & PT31 = 2M vram identity map] 340 * 341 * This makes the lowest part of the VM point to the pagetables. 342 * Hence the lowest 2M in the vm should point to itself, with a few writes 343 * and flushes, other parts of the VM can be used either for copying and 344 * clearing. 345 * 346 * For performance, the kernel reserves PDE's, so about 20 are left 347 * for async VM updates. 348 * 349 * To make it easier to work, each scratch PT is put in slot (1 + PT #) 350 * everywhere, this allows lockless updates to scratch pages by using 351 * the different addresses in VM. 352 */ 353 #define NUM_VMUSA_UNIT_PER_PAGE 32 354 #define VM_SA_UPDATE_UNIT_SIZE (XE_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE) 355 #define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64)) 356 drm_suballoc_manager_init(&m->vm_update_sa, 357 (size_t)(map_ofs / XE_PAGE_SIZE - NUM_KERNEL_PDE) * 358 NUM_VMUSA_UNIT_PER_PAGE, 0); 359 360 m->pt_bo = bo; 361 return 0; 362 } 363 364 /* 365 * Including the reserved copy engine is required to avoid deadlocks due to 366 * migrate jobs servicing the faults gets stuck behind the job that faulted. 367 */ 368 static u32 xe_migrate_usm_logical_mask(struct xe_gt *gt) 369 { 370 u32 logical_mask = 0; 371 struct xe_hw_engine *hwe; 372 enum xe_hw_engine_id id; 373 374 for_each_hw_engine(hwe, gt, id) { 375 if (hwe->class != XE_ENGINE_CLASS_COPY) 376 continue; 377 378 if (xe_gt_is_usm_hwe(gt, hwe)) 379 logical_mask |= BIT(hwe->logical_instance); 380 } 381 382 return logical_mask; 383 } 384 385 static bool xe_migrate_needs_ccs_emit(struct xe_device *xe) 386 { 387 return xe_device_has_flat_ccs(xe) && !(GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)); 388 } 389 390 /** 391 * xe_migrate_init() - Initialize a migrate context 392 * @tile: Back-pointer to the tile we're initializing for. 393 * 394 * Return: Pointer to a migrate context on success. Error pointer on error. 395 */ 396 struct xe_migrate *xe_migrate_init(struct xe_tile *tile) 397 { 398 struct xe_device *xe = tile_to_xe(tile); 399 struct xe_gt *primary_gt = tile->primary_gt; 400 struct xe_migrate *m; 401 struct xe_vm *vm; 402 int err; 403 404 m = drmm_kzalloc(&xe->drm, sizeof(*m), GFP_KERNEL); 405 if (!m) 406 return ERR_PTR(-ENOMEM); 407 408 m->tile = tile; 409 410 /* Special layout, prepared below.. */ 411 vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | 412 XE_VM_FLAG_SET_TILE_ID(tile)); 413 if (IS_ERR(vm)) 414 return ERR_CAST(vm); 415 416 xe_vm_lock(vm, false); 417 err = xe_migrate_prepare_vm(tile, m, vm); 418 xe_vm_unlock(vm); 419 if (err) { 420 xe_vm_close_and_put(vm); 421 return ERR_PTR(err); 422 } 423 424 if (xe->info.has_usm) { 425 struct xe_hw_engine *hwe = xe_gt_hw_engine(primary_gt, 426 XE_ENGINE_CLASS_COPY, 427 primary_gt->usm.reserved_bcs_instance, 428 false); 429 u32 logical_mask = xe_migrate_usm_logical_mask(primary_gt); 430 431 if (!hwe || !logical_mask) 432 return ERR_PTR(-EINVAL); 433 434 /* 435 * XXX: Currently only reserving 1 (likely slow) BCS instance on 436 * PVC, may want to revisit if performance is needed. 437 */ 438 m->q = xe_exec_queue_create(xe, vm, logical_mask, 1, hwe, 439 EXEC_QUEUE_FLAG_KERNEL | 440 EXEC_QUEUE_FLAG_PERMANENT | 441 EXEC_QUEUE_FLAG_HIGH_PRIORITY, 0); 442 } else { 443 m->q = xe_exec_queue_create_class(xe, primary_gt, vm, 444 XE_ENGINE_CLASS_COPY, 445 EXEC_QUEUE_FLAG_KERNEL | 446 EXEC_QUEUE_FLAG_PERMANENT, 0); 447 } 448 if (IS_ERR(m->q)) { 449 xe_vm_close_and_put(vm); 450 return ERR_CAST(m->q); 451 } 452 453 mutex_init(&m->job_mutex); 454 fs_reclaim_acquire(GFP_KERNEL); 455 might_lock(&m->job_mutex); 456 fs_reclaim_release(GFP_KERNEL); 457 458 err = drmm_add_action_or_reset(&xe->drm, xe_migrate_fini, m); 459 if (err) 460 return ERR_PTR(err); 461 462 if (IS_DGFX(xe)) { 463 if (xe_migrate_needs_ccs_emit(xe)) 464 /* min chunk size corresponds to 4K of CCS Metadata */ 465 m->min_chunk_size = SZ_4K * SZ_64K / 466 xe_device_ccs_bytes(xe, SZ_64K); 467 else 468 /* Somewhat arbitrary to avoid a huge amount of blits */ 469 m->min_chunk_size = SZ_64K; 470 m->min_chunk_size = roundup_pow_of_two(m->min_chunk_size); 471 drm_dbg(&xe->drm, "Migrate min chunk size is 0x%08llx\n", 472 (unsigned long long)m->min_chunk_size); 473 } 474 475 return m; 476 } 477 478 static u64 max_mem_transfer_per_pass(struct xe_device *xe) 479 { 480 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe)) 481 return MAX_CCS_LIMITED_TRANSFER; 482 483 return MAX_PREEMPTDISABLE_TRANSFER; 484 } 485 486 static u64 xe_migrate_res_sizes(struct xe_migrate *m, struct xe_res_cursor *cur) 487 { 488 struct xe_device *xe = tile_to_xe(m->tile); 489 u64 size = min_t(u64, max_mem_transfer_per_pass(xe), cur->remaining); 490 491 if (mem_type_is_vram(cur->mem_type)) { 492 /* 493 * VRAM we want to blit in chunks with sizes aligned to 494 * min_chunk_size in order for the offset to CCS metadata to be 495 * page-aligned. If it's the last chunk it may be smaller. 496 * 497 * Another constraint is that we need to limit the blit to 498 * the VRAM block size, unless size is smaller than 499 * min_chunk_size. 500 */ 501 u64 chunk = max_t(u64, cur->size, m->min_chunk_size); 502 503 size = min_t(u64, size, chunk); 504 if (size > m->min_chunk_size) 505 size = round_down(size, m->min_chunk_size); 506 } 507 508 return size; 509 } 510 511 static bool xe_migrate_allow_identity(u64 size, const struct xe_res_cursor *cur) 512 { 513 /* If the chunk is not fragmented, allow identity map. */ 514 return cur->size >= size; 515 } 516 517 #define PTE_UPDATE_FLAG_IS_VRAM BIT(0) 518 #define PTE_UPDATE_FLAG_IS_COMP_PTE BIT(1) 519 520 static u32 pte_update_size(struct xe_migrate *m, 521 u32 flags, 522 struct ttm_resource *res, 523 struct xe_res_cursor *cur, 524 u64 *L0, u64 *L0_ofs, u32 *L0_pt, 525 u32 cmd_size, u32 pt_ofs, u32 avail_pts) 526 { 527 u32 cmds = 0; 528 bool is_vram = PTE_UPDATE_FLAG_IS_VRAM & flags; 529 bool is_comp_pte = PTE_UPDATE_FLAG_IS_COMP_PTE & flags; 530 531 *L0_pt = pt_ofs; 532 if (is_vram && xe_migrate_allow_identity(*L0, cur)) { 533 /* Offset into identity map. */ 534 *L0_ofs = xe_migrate_vram_ofs(tile_to_xe(m->tile), 535 cur->start + vram_region_gpu_offset(res), 536 is_comp_pte); 537 cmds += cmd_size; 538 } else { 539 /* Clip L0 to available size */ 540 u64 size = min(*L0, (u64)avail_pts * SZ_2M); 541 u32 num_4k_pages = (size + XE_PAGE_SIZE - 1) >> XE_PTE_SHIFT; 542 543 *L0 = size; 544 *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0); 545 546 /* MI_STORE_DATA_IMM */ 547 cmds += 3 * DIV_ROUND_UP(num_4k_pages, MAX_PTE_PER_SDI); 548 549 /* PDE qwords */ 550 cmds += num_4k_pages * 2; 551 552 /* Each chunk has a single blit command */ 553 cmds += cmd_size; 554 } 555 556 return cmds; 557 } 558 559 static void emit_pte(struct xe_migrate *m, 560 struct xe_bb *bb, u32 at_pt, 561 bool is_vram, bool is_comp_pte, 562 struct xe_res_cursor *cur, 563 u32 size, struct ttm_resource *res) 564 { 565 struct xe_device *xe = tile_to_xe(m->tile); 566 struct xe_vm *vm = m->q->vm; 567 u16 pat_index; 568 u32 ptes; 569 u64 ofs = (u64)at_pt * XE_PAGE_SIZE; 570 u64 cur_ofs; 571 572 /* Indirect access needs compression enabled uncached PAT index */ 573 if (GRAPHICS_VERx100(xe) >= 2000) 574 pat_index = is_comp_pte ? xe->pat.idx[XE_CACHE_NONE_COMPRESSION] : 575 xe->pat.idx[XE_CACHE_WB]; 576 else 577 pat_index = xe->pat.idx[XE_CACHE_WB]; 578 579 ptes = DIV_ROUND_UP(size, XE_PAGE_SIZE); 580 581 while (ptes) { 582 u32 chunk = min(MAX_PTE_PER_SDI, ptes); 583 584 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 585 bb->cs[bb->len++] = ofs; 586 bb->cs[bb->len++] = 0; 587 588 cur_ofs = ofs; 589 ofs += chunk * 8; 590 ptes -= chunk; 591 592 while (chunk--) { 593 u64 addr, flags = 0; 594 bool devmem = false; 595 596 addr = xe_res_dma(cur) & PAGE_MASK; 597 if (is_vram) { 598 if (vm->flags & XE_VM_FLAG_64K) { 599 u64 va = cur_ofs * XE_PAGE_SIZE / 8; 600 601 xe_assert(xe, (va & (SZ_64K - 1)) == 602 (addr & (SZ_64K - 1))); 603 604 flags |= XE_PTE_PS64; 605 } 606 607 addr += vram_region_gpu_offset(res); 608 devmem = true; 609 } 610 611 addr = vm->pt_ops->pte_encode_addr(m->tile->xe, 612 addr, pat_index, 613 0, devmem, flags); 614 bb->cs[bb->len++] = lower_32_bits(addr); 615 bb->cs[bb->len++] = upper_32_bits(addr); 616 617 xe_res_next(cur, min_t(u32, size, PAGE_SIZE)); 618 cur_ofs += 8; 619 } 620 } 621 } 622 623 #define EMIT_COPY_CCS_DW 5 624 static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, 625 u64 dst_ofs, bool dst_is_indirect, 626 u64 src_ofs, bool src_is_indirect, 627 u32 size) 628 { 629 struct xe_device *xe = gt_to_xe(gt); 630 u32 *cs = bb->cs + bb->len; 631 u32 num_ccs_blks; 632 u32 num_pages; 633 u32 ccs_copy_size; 634 u32 mocs; 635 636 if (GRAPHICS_VERx100(xe) >= 2000) { 637 num_pages = DIV_ROUND_UP(size, XE_PAGE_SIZE); 638 xe_gt_assert(gt, FIELD_FIT(XE2_CCS_SIZE_MASK, num_pages - 1)); 639 640 ccs_copy_size = REG_FIELD_PREP(XE2_CCS_SIZE_MASK, num_pages - 1); 641 mocs = FIELD_PREP(XE2_XY_CTRL_SURF_MOCS_INDEX_MASK, gt->mocs.uc_index); 642 643 } else { 644 num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size), 645 NUM_CCS_BYTES_PER_BLOCK); 646 xe_gt_assert(gt, FIELD_FIT(CCS_SIZE_MASK, num_ccs_blks - 1)); 647 648 ccs_copy_size = REG_FIELD_PREP(CCS_SIZE_MASK, num_ccs_blks - 1); 649 mocs = FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, gt->mocs.uc_index); 650 } 651 652 *cs++ = XY_CTRL_SURF_COPY_BLT | 653 (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT | 654 (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT | 655 ccs_copy_size; 656 *cs++ = lower_32_bits(src_ofs); 657 *cs++ = upper_32_bits(src_ofs) | mocs; 658 *cs++ = lower_32_bits(dst_ofs); 659 *cs++ = upper_32_bits(dst_ofs) | mocs; 660 661 bb->len = cs - bb->cs; 662 } 663 664 #define EMIT_COPY_DW 10 665 static void emit_copy(struct xe_gt *gt, struct xe_bb *bb, 666 u64 src_ofs, u64 dst_ofs, unsigned int size, 667 unsigned int pitch) 668 { 669 struct xe_device *xe = gt_to_xe(gt); 670 u32 mocs = 0; 671 u32 tile_y = 0; 672 673 xe_gt_assert(gt, size / pitch <= S16_MAX); 674 xe_gt_assert(gt, pitch / 4 <= S16_MAX); 675 xe_gt_assert(gt, pitch <= U16_MAX); 676 677 if (GRAPHICS_VER(xe) >= 20) 678 mocs = FIELD_PREP(XE2_XY_FAST_COPY_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index); 679 680 if (GRAPHICS_VERx100(xe) >= 1250) 681 tile_y = XY_FAST_COPY_BLT_D1_SRC_TILE4 | XY_FAST_COPY_BLT_D1_DST_TILE4; 682 683 bb->cs[bb->len++] = XY_FAST_COPY_BLT_CMD | (10 - 2); 684 bb->cs[bb->len++] = XY_FAST_COPY_BLT_DEPTH_32 | pitch | tile_y | mocs; 685 bb->cs[bb->len++] = 0; 686 bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; 687 bb->cs[bb->len++] = lower_32_bits(dst_ofs); 688 bb->cs[bb->len++] = upper_32_bits(dst_ofs); 689 bb->cs[bb->len++] = 0; 690 bb->cs[bb->len++] = pitch | mocs; 691 bb->cs[bb->len++] = lower_32_bits(src_ofs); 692 bb->cs[bb->len++] = upper_32_bits(src_ofs); 693 } 694 695 static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm) 696 { 697 return usm ? m->usm_batch_base_ofs : m->batch_base_ofs; 698 } 699 700 static u32 xe_migrate_ccs_copy(struct xe_migrate *m, 701 struct xe_bb *bb, 702 u64 src_ofs, bool src_is_indirect, 703 u64 dst_ofs, bool dst_is_indirect, u32 dst_size, 704 u64 ccs_ofs, bool copy_ccs) 705 { 706 struct xe_gt *gt = m->tile->primary_gt; 707 u32 flush_flags = 0; 708 709 if (!copy_ccs && dst_is_indirect) { 710 /* 711 * If the src is already in vram, then it should already 712 * have been cleared by us, or has been populated by the 713 * user. Make sure we copy the CCS aux state as-is. 714 * 715 * Otherwise if the bo doesn't have any CCS metadata attached, 716 * we still need to clear it for security reasons. 717 */ 718 u64 ccs_src_ofs = src_is_indirect ? src_ofs : m->cleared_mem_ofs; 719 720 emit_copy_ccs(gt, bb, 721 dst_ofs, true, 722 ccs_src_ofs, src_is_indirect, dst_size); 723 724 flush_flags = MI_FLUSH_DW_CCS; 725 } else if (copy_ccs) { 726 if (!src_is_indirect) 727 src_ofs = ccs_ofs; 728 else if (!dst_is_indirect) 729 dst_ofs = ccs_ofs; 730 731 xe_gt_assert(gt, src_is_indirect || dst_is_indirect); 732 733 emit_copy_ccs(gt, bb, dst_ofs, dst_is_indirect, src_ofs, 734 src_is_indirect, dst_size); 735 if (dst_is_indirect) 736 flush_flags = MI_FLUSH_DW_CCS; 737 } 738 739 return flush_flags; 740 } 741 742 /** 743 * xe_migrate_copy() - Copy content of TTM resources. 744 * @m: The migration context. 745 * @src_bo: The buffer object @src is currently bound to. 746 * @dst_bo: If copying between resources created for the same bo, set this to 747 * the same value as @src_bo. If copying between buffer objects, set it to 748 * the buffer object @dst is currently bound to. 749 * @src: The source TTM resource. 750 * @dst: The dst TTM resource. 751 * @copy_only_ccs: If true copy only CCS metadata 752 * 753 * Copies the contents of @src to @dst: On flat CCS devices, 754 * the CCS metadata is copied as well if needed, or if not present, 755 * the CCS metadata of @dst is cleared for security reasons. 756 * 757 * Return: Pointer to a dma_fence representing the last copy batch, or 758 * an error pointer on failure. If there is a failure, any copy operation 759 * started by the function call has been synced. 760 */ 761 struct dma_fence *xe_migrate_copy(struct xe_migrate *m, 762 struct xe_bo *src_bo, 763 struct xe_bo *dst_bo, 764 struct ttm_resource *src, 765 struct ttm_resource *dst, 766 bool copy_only_ccs) 767 { 768 struct xe_gt *gt = m->tile->primary_gt; 769 struct xe_device *xe = gt_to_xe(gt); 770 struct dma_fence *fence = NULL; 771 u64 size = src_bo->size; 772 struct xe_res_cursor src_it, dst_it, ccs_it; 773 u64 src_L0_ofs, dst_L0_ofs; 774 u32 src_L0_pt, dst_L0_pt; 775 u64 src_L0, dst_L0; 776 int pass = 0; 777 int err; 778 bool src_is_pltt = src->mem_type == XE_PL_TT; 779 bool dst_is_pltt = dst->mem_type == XE_PL_TT; 780 bool src_is_vram = mem_type_is_vram(src->mem_type); 781 bool dst_is_vram = mem_type_is_vram(dst->mem_type); 782 bool copy_ccs = xe_device_has_flat_ccs(xe) && 783 xe_bo_needs_ccs_pages(src_bo) && xe_bo_needs_ccs_pages(dst_bo); 784 bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram); 785 bool use_comp_pat = xe_device_has_flat_ccs(xe) && 786 GRAPHICS_VER(xe) >= 20 && src_is_vram && !dst_is_vram; 787 788 /* Copying CCS between two different BOs is not supported yet. */ 789 if (XE_WARN_ON(copy_ccs && src_bo != dst_bo)) 790 return ERR_PTR(-EINVAL); 791 792 if (src_bo != dst_bo && XE_WARN_ON(src_bo->size != dst_bo->size)) 793 return ERR_PTR(-EINVAL); 794 795 if (!src_is_vram) 796 xe_res_first_sg(xe_bo_sg(src_bo), 0, size, &src_it); 797 else 798 xe_res_first(src, 0, size, &src_it); 799 if (!dst_is_vram) 800 xe_res_first_sg(xe_bo_sg(dst_bo), 0, size, &dst_it); 801 else 802 xe_res_first(dst, 0, size, &dst_it); 803 804 if (copy_system_ccs) 805 xe_res_first_sg(xe_bo_sg(src_bo), xe_bo_ccs_pages_start(src_bo), 806 PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), 807 &ccs_it); 808 809 while (size) { 810 u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */ 811 struct xe_sched_job *job; 812 struct xe_bb *bb; 813 u32 flush_flags = 0; 814 u32 update_idx; 815 u64 ccs_ofs, ccs_size; 816 u32 ccs_pt; 817 u32 pte_flags; 818 819 bool usm = xe->info.has_usm; 820 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 821 822 src_L0 = xe_migrate_res_sizes(m, &src_it); 823 dst_L0 = xe_migrate_res_sizes(m, &dst_it); 824 825 drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n", 826 pass++, src_L0, dst_L0); 827 828 src_L0 = min(src_L0, dst_L0); 829 830 pte_flags = src_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; 831 pte_flags |= use_comp_pat ? PTE_UPDATE_FLAG_IS_COMP_PTE : 0; 832 batch_size += pte_update_size(m, pte_flags, src, &src_it, &src_L0, 833 &src_L0_ofs, &src_L0_pt, 0, 0, 834 avail_pts); 835 836 pte_flags = dst_is_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; 837 batch_size += pte_update_size(m, pte_flags, dst, &dst_it, &src_L0, 838 &dst_L0_ofs, &dst_L0_pt, 0, 839 avail_pts, avail_pts); 840 841 if (copy_system_ccs) { 842 ccs_size = xe_device_ccs_bytes(xe, src_L0); 843 batch_size += pte_update_size(m, 0, NULL, &ccs_it, &ccs_size, 844 &ccs_ofs, &ccs_pt, 0, 845 2 * avail_pts, 846 avail_pts); 847 xe_assert(xe, IS_ALIGNED(ccs_it.start, PAGE_SIZE)); 848 } 849 850 /* Add copy commands size here */ 851 batch_size += ((copy_only_ccs) ? 0 : EMIT_COPY_DW) + 852 ((xe_migrate_needs_ccs_emit(xe) ? EMIT_COPY_CCS_DW : 0)); 853 854 bb = xe_bb_new(gt, batch_size, usm); 855 if (IS_ERR(bb)) { 856 err = PTR_ERR(bb); 857 goto err_sync; 858 } 859 860 if (src_is_vram && xe_migrate_allow_identity(src_L0, &src_it)) 861 xe_res_next(&src_it, src_L0); 862 else 863 emit_pte(m, bb, src_L0_pt, src_is_vram, copy_system_ccs, 864 &src_it, src_L0, src); 865 866 if (dst_is_vram && xe_migrate_allow_identity(src_L0, &dst_it)) 867 xe_res_next(&dst_it, src_L0); 868 else 869 emit_pte(m, bb, dst_L0_pt, dst_is_vram, copy_system_ccs, 870 &dst_it, src_L0, dst); 871 872 if (copy_system_ccs) 873 emit_pte(m, bb, ccs_pt, false, false, &ccs_it, ccs_size, src); 874 875 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 876 update_idx = bb->len; 877 878 if (!copy_only_ccs) 879 emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, XE_PAGE_SIZE); 880 881 if (xe_migrate_needs_ccs_emit(xe)) 882 flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, 883 IS_DGFX(xe) ? src_is_vram : src_is_pltt, 884 dst_L0_ofs, 885 IS_DGFX(xe) ? dst_is_vram : dst_is_pltt, 886 src_L0, ccs_ofs, copy_ccs); 887 888 job = xe_bb_create_migration_job(m->q, bb, 889 xe_migrate_batch_base(m, usm), 890 update_idx); 891 if (IS_ERR(job)) { 892 err = PTR_ERR(job); 893 goto err; 894 } 895 896 xe_sched_job_add_migrate_flush(job, flush_flags); 897 if (!fence) { 898 err = xe_sched_job_add_deps(job, src_bo->ttm.base.resv, 899 DMA_RESV_USAGE_BOOKKEEP); 900 if (!err && src_bo != dst_bo) 901 err = xe_sched_job_add_deps(job, dst_bo->ttm.base.resv, 902 DMA_RESV_USAGE_BOOKKEEP); 903 if (err) 904 goto err_job; 905 } 906 907 mutex_lock(&m->job_mutex); 908 xe_sched_job_arm(job); 909 dma_fence_put(fence); 910 fence = dma_fence_get(&job->drm.s_fence->finished); 911 xe_sched_job_push(job); 912 913 dma_fence_put(m->fence); 914 m->fence = dma_fence_get(fence); 915 916 mutex_unlock(&m->job_mutex); 917 918 xe_bb_free(bb, fence); 919 size -= src_L0; 920 continue; 921 922 err_job: 923 xe_sched_job_put(job); 924 err: 925 xe_bb_free(bb, NULL); 926 927 err_sync: 928 /* Sync partial copy if any. FIXME: under job_mutex? */ 929 if (fence) { 930 dma_fence_wait(fence, false); 931 dma_fence_put(fence); 932 } 933 934 return ERR_PTR(err); 935 } 936 937 return fence; 938 } 939 940 static void emit_clear_link_copy(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 941 u32 size, u32 pitch) 942 { 943 struct xe_device *xe = gt_to_xe(gt); 944 u32 *cs = bb->cs + bb->len; 945 u32 len = PVC_MEM_SET_CMD_LEN_DW; 946 947 *cs++ = PVC_MEM_SET_CMD | PVC_MEM_SET_MATRIX | (len - 2); 948 *cs++ = pitch - 1; 949 *cs++ = (size / pitch) - 1; 950 *cs++ = pitch - 1; 951 *cs++ = lower_32_bits(src_ofs); 952 *cs++ = upper_32_bits(src_ofs); 953 if (GRAPHICS_VERx100(xe) >= 2000) 954 *cs++ = FIELD_PREP(XE2_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 955 else 956 *cs++ = FIELD_PREP(PVC_MEM_SET_MOCS_INDEX_MASK, gt->mocs.uc_index); 957 958 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 959 960 bb->len += len; 961 } 962 963 static void emit_clear_main_copy(struct xe_gt *gt, struct xe_bb *bb, 964 u64 src_ofs, u32 size, u32 pitch, bool is_vram) 965 { 966 struct xe_device *xe = gt_to_xe(gt); 967 u32 *cs = bb->cs + bb->len; 968 u32 len = XY_FAST_COLOR_BLT_DW; 969 970 if (GRAPHICS_VERx100(xe) < 1250) 971 len = 11; 972 973 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | 974 (len - 2); 975 if (GRAPHICS_VERx100(xe) >= 2000) 976 *cs++ = FIELD_PREP(XE2_XY_FAST_COLOR_BLT_MOCS_INDEX_MASK, gt->mocs.uc_index) | 977 (pitch - 1); 978 else 979 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, gt->mocs.uc_index) | 980 (pitch - 1); 981 *cs++ = 0; 982 *cs++ = (size / pitch) << 16 | pitch / 4; 983 *cs++ = lower_32_bits(src_ofs); 984 *cs++ = upper_32_bits(src_ofs); 985 *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; 986 *cs++ = 0; 987 *cs++ = 0; 988 *cs++ = 0; 989 *cs++ = 0; 990 991 if (len > 11) { 992 *cs++ = 0; 993 *cs++ = 0; 994 *cs++ = 0; 995 *cs++ = 0; 996 *cs++ = 0; 997 } 998 999 xe_gt_assert(gt, cs - bb->cs == len + bb->len); 1000 1001 bb->len += len; 1002 } 1003 1004 static bool has_service_copy_support(struct xe_gt *gt) 1005 { 1006 /* 1007 * What we care about is whether the architecture was designed with 1008 * service copy functionality (specifically the new MEM_SET / MEM_COPY 1009 * instructions) so check the architectural engine list rather than the 1010 * actual list since these instructions are usable on BCS0 even if 1011 * all of the actual service copy engines (BCS1-BCS8) have been fused 1012 * off. 1013 */ 1014 return gt->info.engine_mask & GENMASK(XE_HW_ENGINE_BCS8, 1015 XE_HW_ENGINE_BCS1); 1016 } 1017 1018 static u32 emit_clear_cmd_len(struct xe_gt *gt) 1019 { 1020 if (has_service_copy_support(gt)) 1021 return PVC_MEM_SET_CMD_LEN_DW; 1022 else 1023 return XY_FAST_COLOR_BLT_DW; 1024 } 1025 1026 static void emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, 1027 u32 size, u32 pitch, bool is_vram) 1028 { 1029 if (has_service_copy_support(gt)) 1030 emit_clear_link_copy(gt, bb, src_ofs, size, pitch); 1031 else 1032 emit_clear_main_copy(gt, bb, src_ofs, size, pitch, 1033 is_vram); 1034 } 1035 1036 /** 1037 * xe_migrate_clear() - Copy content of TTM resources. 1038 * @m: The migration context. 1039 * @bo: The buffer object @dst is currently bound to. 1040 * @dst: The dst TTM resource to be cleared. 1041 * @clear_flags: flags to specify which data to clear: CCS, BO, or both. 1042 * 1043 * Clear the contents of @dst to zero when XE_MIGRATE_CLEAR_FLAG_BO_DATA is set. 1044 * On flat CCS devices, the CCS metadata is cleared to zero with XE_MIGRATE_CLEAR_FLAG_CCS_DATA. 1045 * Set XE_MIGRATE_CLEAR_FLAG_FULL to clear bo as well as CCS metadata. 1046 * TODO: Eliminate the @bo argument. 1047 * 1048 * Return: Pointer to a dma_fence representing the last clear batch, or 1049 * an error pointer on failure. If there is a failure, any clear operation 1050 * started by the function call has been synced. 1051 */ 1052 struct dma_fence *xe_migrate_clear(struct xe_migrate *m, 1053 struct xe_bo *bo, 1054 struct ttm_resource *dst, 1055 u32 clear_flags) 1056 { 1057 bool clear_vram = mem_type_is_vram(dst->mem_type); 1058 bool clear_bo_data = XE_MIGRATE_CLEAR_FLAG_BO_DATA & clear_flags; 1059 bool clear_ccs = XE_MIGRATE_CLEAR_FLAG_CCS_DATA & clear_flags; 1060 struct xe_gt *gt = m->tile->primary_gt; 1061 struct xe_device *xe = gt_to_xe(gt); 1062 bool clear_only_system_ccs = false; 1063 struct dma_fence *fence = NULL; 1064 u64 size = bo->size; 1065 struct xe_res_cursor src_it; 1066 struct ttm_resource *src = dst; 1067 int err; 1068 1069 if (WARN_ON(!clear_bo_data && !clear_ccs)) 1070 return NULL; 1071 1072 if (!clear_bo_data && clear_ccs && !IS_DGFX(xe)) 1073 clear_only_system_ccs = true; 1074 1075 if (!clear_vram) 1076 xe_res_first_sg(xe_bo_sg(bo), 0, bo->size, &src_it); 1077 else 1078 xe_res_first(src, 0, bo->size, &src_it); 1079 1080 while (size) { 1081 u64 clear_L0_ofs; 1082 u32 clear_L0_pt; 1083 u32 flush_flags = 0; 1084 u64 clear_L0; 1085 struct xe_sched_job *job; 1086 struct xe_bb *bb; 1087 u32 batch_size, update_idx; 1088 u32 pte_flags; 1089 1090 bool usm = xe->info.has_usm; 1091 u32 avail_pts = max_mem_transfer_per_pass(xe) / LEVEL0_PAGE_TABLE_ENCODE_SIZE; 1092 1093 clear_L0 = xe_migrate_res_sizes(m, &src_it); 1094 1095 /* Calculate final sizes and batch size.. */ 1096 pte_flags = clear_vram ? PTE_UPDATE_FLAG_IS_VRAM : 0; 1097 batch_size = 2 + 1098 pte_update_size(m, pte_flags, src, &src_it, 1099 &clear_L0, &clear_L0_ofs, &clear_L0_pt, 1100 clear_bo_data ? emit_clear_cmd_len(gt) : 0, 0, 1101 avail_pts); 1102 1103 if (xe_migrate_needs_ccs_emit(xe)) 1104 batch_size += EMIT_COPY_CCS_DW; 1105 1106 /* Clear commands */ 1107 1108 if (WARN_ON_ONCE(!clear_L0)) 1109 break; 1110 1111 bb = xe_bb_new(gt, batch_size, usm); 1112 if (IS_ERR(bb)) { 1113 err = PTR_ERR(bb); 1114 goto err_sync; 1115 } 1116 1117 size -= clear_L0; 1118 /* Preemption is enabled again by the ring ops. */ 1119 if (clear_vram && xe_migrate_allow_identity(clear_L0, &src_it)) 1120 xe_res_next(&src_it, clear_L0); 1121 else 1122 emit_pte(m, bb, clear_L0_pt, clear_vram, clear_only_system_ccs, 1123 &src_it, clear_L0, dst); 1124 1125 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1126 update_idx = bb->len; 1127 1128 if (clear_bo_data) 1129 emit_clear(gt, bb, clear_L0_ofs, clear_L0, XE_PAGE_SIZE, clear_vram); 1130 1131 if (xe_migrate_needs_ccs_emit(xe)) { 1132 emit_copy_ccs(gt, bb, clear_L0_ofs, true, 1133 m->cleared_mem_ofs, false, clear_L0); 1134 flush_flags = MI_FLUSH_DW_CCS; 1135 } 1136 1137 job = xe_bb_create_migration_job(m->q, bb, 1138 xe_migrate_batch_base(m, usm), 1139 update_idx); 1140 if (IS_ERR(job)) { 1141 err = PTR_ERR(job); 1142 goto err; 1143 } 1144 1145 xe_sched_job_add_migrate_flush(job, flush_flags); 1146 if (!fence) { 1147 /* 1148 * There can't be anything userspace related at this 1149 * point, so we just need to respect any potential move 1150 * fences, which are always tracked as 1151 * DMA_RESV_USAGE_KERNEL. 1152 */ 1153 err = xe_sched_job_add_deps(job, bo->ttm.base.resv, 1154 DMA_RESV_USAGE_KERNEL); 1155 if (err) 1156 goto err_job; 1157 } 1158 1159 mutex_lock(&m->job_mutex); 1160 xe_sched_job_arm(job); 1161 dma_fence_put(fence); 1162 fence = dma_fence_get(&job->drm.s_fence->finished); 1163 xe_sched_job_push(job); 1164 1165 dma_fence_put(m->fence); 1166 m->fence = dma_fence_get(fence); 1167 1168 mutex_unlock(&m->job_mutex); 1169 1170 xe_bb_free(bb, fence); 1171 continue; 1172 1173 err_job: 1174 xe_sched_job_put(job); 1175 err: 1176 xe_bb_free(bb, NULL); 1177 err_sync: 1178 /* Sync partial copies if any. FIXME: job_mutex? */ 1179 if (fence) { 1180 dma_fence_wait(m->fence, false); 1181 dma_fence_put(fence); 1182 } 1183 1184 return ERR_PTR(err); 1185 } 1186 1187 if (clear_ccs) 1188 bo->ccs_cleared = true; 1189 1190 return fence; 1191 } 1192 1193 static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs, 1194 const struct xe_vm_pgtable_update_op *pt_op, 1195 const struct xe_vm_pgtable_update *update, 1196 struct xe_migrate_pt_update *pt_update) 1197 { 1198 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1199 u32 chunk; 1200 u32 ofs = update->ofs, size = update->qwords; 1201 1202 /* 1203 * If we have 512 entries (max), we would populate it ourselves, 1204 * and update the PDE above it to the new pointer. 1205 * The only time this can only happen if we have to update the top 1206 * PDE. This requires a BO that is almost vm->size big. 1207 * 1208 * This shouldn't be possible in practice.. might change when 16K 1209 * pages are used. Hence the assert. 1210 */ 1211 xe_tile_assert(tile, update->qwords < MAX_NUM_PTE); 1212 if (!ppgtt_ofs) 1213 ppgtt_ofs = xe_migrate_vram_ofs(tile_to_xe(tile), 1214 xe_bo_addr(update->pt_bo, 0, 1215 XE_PAGE_SIZE), false); 1216 1217 do { 1218 u64 addr = ppgtt_ofs + ofs * 8; 1219 1220 chunk = min(size, MAX_PTE_PER_SDI); 1221 1222 /* Ensure populatefn can do memset64 by aligning bb->cs */ 1223 if (!(bb->len & 1)) 1224 bb->cs[bb->len++] = MI_NOOP; 1225 1226 bb->cs[bb->len++] = MI_STORE_DATA_IMM | MI_SDI_NUM_QW(chunk); 1227 bb->cs[bb->len++] = lower_32_bits(addr); 1228 bb->cs[bb->len++] = upper_32_bits(addr); 1229 if (pt_op->bind) 1230 ops->populate(pt_update, tile, NULL, bb->cs + bb->len, 1231 ofs, chunk, update); 1232 else 1233 ops->clear(pt_update, tile, NULL, bb->cs + bb->len, 1234 ofs, chunk, update); 1235 1236 bb->len += chunk * 2; 1237 ofs += chunk; 1238 size -= chunk; 1239 } while (size); 1240 } 1241 1242 struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) 1243 { 1244 return xe_vm_get(m->q->vm); 1245 } 1246 1247 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1248 struct migrate_test_params { 1249 struct xe_test_priv base; 1250 bool force_gpu; 1251 }; 1252 1253 #define to_migrate_test_params(_priv) \ 1254 container_of(_priv, struct migrate_test_params, base) 1255 #endif 1256 1257 static struct dma_fence * 1258 xe_migrate_update_pgtables_cpu(struct xe_migrate *m, 1259 struct xe_migrate_pt_update *pt_update) 1260 { 1261 XE_TEST_DECLARE(struct migrate_test_params *test = 1262 to_migrate_test_params 1263 (xe_cur_kunit_priv(XE_TEST_LIVE_MIGRATE));) 1264 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1265 struct xe_vm *vm = pt_update->vops->vm; 1266 struct xe_vm_pgtable_update_ops *pt_update_ops = 1267 &pt_update->vops->pt_update_ops[pt_update->tile_id]; 1268 int err; 1269 u32 i, j; 1270 1271 if (XE_TEST_ONLY(test && test->force_gpu)) 1272 return ERR_PTR(-ETIME); 1273 1274 if (ops->pre_commit) { 1275 pt_update->job = NULL; 1276 err = ops->pre_commit(pt_update); 1277 if (err) 1278 return ERR_PTR(err); 1279 } 1280 1281 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1282 const struct xe_vm_pgtable_update_op *pt_op = 1283 &pt_update_ops->ops[i]; 1284 1285 for (j = 0; j < pt_op->num_entries; j++) { 1286 const struct xe_vm_pgtable_update *update = 1287 &pt_op->entries[j]; 1288 1289 if (pt_op->bind) 1290 ops->populate(pt_update, m->tile, 1291 &update->pt_bo->vmap, NULL, 1292 update->ofs, update->qwords, 1293 update); 1294 else 1295 ops->clear(pt_update, m->tile, 1296 &update->pt_bo->vmap, NULL, 1297 update->ofs, update->qwords, update); 1298 } 1299 } 1300 1301 trace_xe_vm_cpu_bind(vm); 1302 xe_device_wmb(vm->xe); 1303 1304 return dma_fence_get_stub(); 1305 } 1306 1307 static struct dma_fence * 1308 __xe_migrate_update_pgtables(struct xe_migrate *m, 1309 struct xe_migrate_pt_update *pt_update, 1310 struct xe_vm_pgtable_update_ops *pt_update_ops) 1311 { 1312 const struct xe_migrate_pt_update_ops *ops = pt_update->ops; 1313 struct xe_tile *tile = m->tile; 1314 struct xe_gt *gt = tile->primary_gt; 1315 struct xe_device *xe = tile_to_xe(tile); 1316 struct xe_sched_job *job; 1317 struct dma_fence *fence; 1318 struct drm_suballoc *sa_bo = NULL; 1319 struct xe_bb *bb; 1320 u32 i, j, batch_size = 0, ppgtt_ofs, update_idx, page_ofs = 0; 1321 u32 num_updates = 0, current_update = 0; 1322 u64 addr; 1323 int err = 0; 1324 bool is_migrate = pt_update_ops->q == m->q; 1325 bool usm = is_migrate && xe->info.has_usm; 1326 1327 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1328 struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[i]; 1329 struct xe_vm_pgtable_update *updates = pt_op->entries; 1330 1331 num_updates += pt_op->num_entries; 1332 for (j = 0; j < pt_op->num_entries; ++j) { 1333 u32 num_cmds = DIV_ROUND_UP(updates[j].qwords, 1334 MAX_PTE_PER_SDI); 1335 1336 /* align noop + MI_STORE_DATA_IMM cmd prefix */ 1337 batch_size += 4 * num_cmds + updates[j].qwords * 2; 1338 } 1339 } 1340 1341 /* fixed + PTE entries */ 1342 if (IS_DGFX(xe)) 1343 batch_size += 2; 1344 else 1345 batch_size += 6 * (num_updates / MAX_PTE_PER_SDI + 1) + 1346 num_updates * 2; 1347 1348 bb = xe_bb_new(gt, batch_size, usm); 1349 if (IS_ERR(bb)) 1350 return ERR_CAST(bb); 1351 1352 /* For sysmem PTE's, need to map them in our hole.. */ 1353 if (!IS_DGFX(xe)) { 1354 u16 pat_index = xe->pat.idx[XE_CACHE_WB]; 1355 u32 ptes, ofs; 1356 1357 ppgtt_ofs = NUM_KERNEL_PDE - 1; 1358 if (!is_migrate) { 1359 u32 num_units = DIV_ROUND_UP(num_updates, 1360 NUM_VMUSA_WRITES_PER_UNIT); 1361 1362 if (num_units > m->vm_update_sa.size) { 1363 err = -ENOBUFS; 1364 goto err_bb; 1365 } 1366 sa_bo = drm_suballoc_new(&m->vm_update_sa, num_units, 1367 GFP_KERNEL, true, 0); 1368 if (IS_ERR(sa_bo)) { 1369 err = PTR_ERR(sa_bo); 1370 goto err_bb; 1371 } 1372 1373 ppgtt_ofs = NUM_KERNEL_PDE + 1374 (drm_suballoc_soffset(sa_bo) / 1375 NUM_VMUSA_UNIT_PER_PAGE); 1376 page_ofs = (drm_suballoc_soffset(sa_bo) % 1377 NUM_VMUSA_UNIT_PER_PAGE) * 1378 VM_SA_UPDATE_UNIT_SIZE; 1379 } 1380 1381 /* Map our PT's to gtt */ 1382 i = 0; 1383 j = 0; 1384 ptes = num_updates; 1385 ofs = ppgtt_ofs * XE_PAGE_SIZE + page_ofs; 1386 while (ptes) { 1387 u32 chunk = min(MAX_PTE_PER_SDI, ptes); 1388 u32 idx = 0; 1389 1390 bb->cs[bb->len++] = MI_STORE_DATA_IMM | 1391 MI_SDI_NUM_QW(chunk); 1392 bb->cs[bb->len++] = ofs; 1393 bb->cs[bb->len++] = 0; /* upper_32_bits */ 1394 1395 for (; i < pt_update_ops->num_ops; ++i) { 1396 struct xe_vm_pgtable_update_op *pt_op = 1397 &pt_update_ops->ops[i]; 1398 struct xe_vm_pgtable_update *updates = pt_op->entries; 1399 1400 for (; j < pt_op->num_entries; ++j, ++current_update, ++idx) { 1401 struct xe_vm *vm = pt_update->vops->vm; 1402 struct xe_bo *pt_bo = updates[j].pt_bo; 1403 1404 if (idx == chunk) 1405 goto next_cmd; 1406 1407 xe_tile_assert(tile, pt_bo->size == SZ_4K); 1408 1409 /* Map a PT at most once */ 1410 if (pt_bo->update_index < 0) 1411 pt_bo->update_index = current_update; 1412 1413 addr = vm->pt_ops->pte_encode_bo(pt_bo, 0, 1414 pat_index, 0); 1415 bb->cs[bb->len++] = lower_32_bits(addr); 1416 bb->cs[bb->len++] = upper_32_bits(addr); 1417 } 1418 1419 j = 0; 1420 } 1421 1422 next_cmd: 1423 ptes -= chunk; 1424 ofs += chunk * sizeof(u64); 1425 } 1426 1427 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1428 update_idx = bb->len; 1429 1430 addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + 1431 (page_ofs / sizeof(u64)) * XE_PAGE_SIZE; 1432 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1433 struct xe_vm_pgtable_update_op *pt_op = 1434 &pt_update_ops->ops[i]; 1435 struct xe_vm_pgtable_update *updates = pt_op->entries; 1436 1437 for (j = 0; j < pt_op->num_entries; ++j) { 1438 struct xe_bo *pt_bo = updates[j].pt_bo; 1439 1440 write_pgtable(tile, bb, addr + 1441 pt_bo->update_index * XE_PAGE_SIZE, 1442 pt_op, &updates[j], pt_update); 1443 } 1444 } 1445 } else { 1446 /* phys pages, no preamble required */ 1447 bb->cs[bb->len++] = MI_BATCH_BUFFER_END; 1448 update_idx = bb->len; 1449 1450 for (i = 0; i < pt_update_ops->num_ops; ++i) { 1451 struct xe_vm_pgtable_update_op *pt_op = 1452 &pt_update_ops->ops[i]; 1453 struct xe_vm_pgtable_update *updates = pt_op->entries; 1454 1455 for (j = 0; j < pt_op->num_entries; ++j) 1456 write_pgtable(tile, bb, 0, pt_op, &updates[j], 1457 pt_update); 1458 } 1459 } 1460 1461 job = xe_bb_create_migration_job(pt_update_ops->q, bb, 1462 xe_migrate_batch_base(m, usm), 1463 update_idx); 1464 if (IS_ERR(job)) { 1465 err = PTR_ERR(job); 1466 goto err_sa; 1467 } 1468 1469 if (ops->pre_commit) { 1470 pt_update->job = job; 1471 err = ops->pre_commit(pt_update); 1472 if (err) 1473 goto err_job; 1474 } 1475 if (is_migrate) 1476 mutex_lock(&m->job_mutex); 1477 1478 xe_sched_job_arm(job); 1479 fence = dma_fence_get(&job->drm.s_fence->finished); 1480 xe_sched_job_push(job); 1481 1482 if (is_migrate) 1483 mutex_unlock(&m->job_mutex); 1484 1485 xe_bb_free(bb, fence); 1486 drm_suballoc_free(sa_bo, fence); 1487 1488 return fence; 1489 1490 err_job: 1491 xe_sched_job_put(job); 1492 err_sa: 1493 drm_suballoc_free(sa_bo, NULL); 1494 err_bb: 1495 xe_bb_free(bb, NULL); 1496 return ERR_PTR(err); 1497 } 1498 1499 /** 1500 * xe_migrate_update_pgtables() - Pipelined page-table update 1501 * @m: The migrate context. 1502 * @pt_update: PT update arguments 1503 * 1504 * Perform a pipelined page-table update. The update descriptors are typically 1505 * built under the same lock critical section as a call to this function. If 1506 * using the default engine for the updates, they will be performed in the 1507 * order they grab the job_mutex. If different engines are used, external 1508 * synchronization is needed for overlapping updates to maintain page-table 1509 * consistency. Note that the meaing of "overlapping" is that the updates 1510 * touch the same page-table, which might be a higher-level page-directory. 1511 * If no pipelining is needed, then updates may be performed by the cpu. 1512 * 1513 * Return: A dma_fence that, when signaled, indicates the update completion. 1514 */ 1515 struct dma_fence * 1516 xe_migrate_update_pgtables(struct xe_migrate *m, 1517 struct xe_migrate_pt_update *pt_update) 1518 1519 { 1520 struct xe_vm_pgtable_update_ops *pt_update_ops = 1521 &pt_update->vops->pt_update_ops[pt_update->tile_id]; 1522 struct dma_fence *fence; 1523 1524 fence = xe_migrate_update_pgtables_cpu(m, pt_update); 1525 1526 /* -ETIME indicates a job is needed, anything else is legit error */ 1527 if (!IS_ERR(fence) || PTR_ERR(fence) != -ETIME) 1528 return fence; 1529 1530 return __xe_migrate_update_pgtables(m, pt_update, pt_update_ops); 1531 } 1532 1533 /** 1534 * xe_migrate_wait() - Complete all operations using the xe_migrate context 1535 * @m: Migrate context to wait for. 1536 * 1537 * Waits until the GPU no longer uses the migrate context's default engine 1538 * or its page-table objects. FIXME: What about separate page-table update 1539 * engines? 1540 */ 1541 void xe_migrate_wait(struct xe_migrate *m) 1542 { 1543 if (m->fence) 1544 dma_fence_wait(m->fence, false); 1545 } 1546 1547 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) 1548 #include "tests/xe_migrate.c" 1549 #endif 1550