Lines Matching +full:cs +full:- +full:3
1 // SPDX-License-Identifier: MIT
33 GEM_BUG_ON(engine->class != COPY_ENGINE_CLASS); in engine_supports_migration()
48 vm->insert_page(vm, 0, d->offset, in xehp_toggle_pdes()
49 i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE), in xehp_toggle_pdes()
51 GEM_BUG_ON(!pt->is_compact); in xehp_toggle_pdes()
52 d->offset += SZ_2M; in xehp_toggle_pdes()
68 vm->insert_page(vm, px_dma(pt), d->offset, in xehp_insert_pte()
69 i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE), in xehp_insert_pte()
71 d->offset += SZ_64K; in xehp_insert_pte()
80 vm->insert_page(vm, px_dma(pt), d->offset, in insert_pte()
81 i915_gem_get_pat_index(vm->i915, I915_CACHE_NONE), in insert_pte()
82 i915_gem_object_is_lmem(pt->base) ? PTE_LM : 0); in insert_pte()
83 d->offset += PAGE_SIZE; in insert_pte()
96 * to pre-allocate the page directories for the migration VM, this in migrate_vm()
107 * [0, CHUNK_SZ) -> first object in migrate_vm()
108 * [CHUNK_SZ, 2 * CHUNK_SZ) -> second object in migrate_vm()
109 * [2 * CHUNK_SZ, 2 * CHUNK_SZ + 2 * CHUNK_SZ >> 9] -> PTE in migrate_vm()
114 * i.e. within the same non-preemptible window so that we do not switch in migrate_vm()
119 * first is reserved for mapping system-memory, and that just uses the in migrate_vm()
125 * compact layout for each of these page-tables, that fall within the in migrate_vm()
126 * [CHUNK_SIZE, 3 * CHUNK_SIZE) range. in migrate_vm()
130 * [0, CHUNK_SZ) -> first window/object, maps smem in migrate_vm()
131 * [CHUNK_SZ, 2 * CHUNK_SZ) -> second window/object, maps lmem src in migrate_vm()
132 * [2 * CHUNK_SZ, 3 * CHUNK_SZ) -> third window/object, maps lmem dst in migrate_vm()
142 * [3 * CHUNK_SZ, 3 * CHUNK_SZ + ((3 * CHUNK_SZ / SZ_2M) * SZ_64K)] -> PTE in migrate_vm()
149 if (!vm->vm.allocate_va_range || !vm->vm.foreach) { in migrate_vm()
150 err = -ENODEV; in migrate_vm()
154 if (HAS_64K_PAGES(gt->i915)) in migrate_vm()
161 for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) { in migrate_vm()
168 engine = gt->engine_class[COPY_ENGINE_CLASS][i]; in migrate_vm()
176 if (HAS_64K_PAGES(gt->i915)) in migrate_vm()
177 sz = 3 * CHUNK_SZ; in migrate_vm()
186 if (HAS_64K_PAGES(gt->i915)) in migrate_vm()
191 err = i915_vm_alloc_pt_stash(&vm->vm, &stash, sz); in migrate_vm()
196 err = i915_vm_lock_objects(&vm->vm, &ww); in migrate_vm()
199 err = i915_vm_map_pt_stash(&vm->vm, &stash); in migrate_vm()
203 vm->vm.allocate_va_range(&vm->vm, &stash, base, sz); in migrate_vm()
205 i915_vm_free_pt_stash(&vm->vm, &stash); in migrate_vm()
210 if (HAS_64K_PAGES(gt->i915)) { in migrate_vm()
211 vm->vm.foreach(&vm->vm, base, d.offset - base, in migrate_vm()
214 vm->vm.foreach(&vm->vm, in migrate_vm()
219 vm->vm.foreach(&vm->vm, base, d.offset - base, in migrate_vm()
224 return &vm->vm; in migrate_vm()
227 i915_vm_put(&vm->vm); in migrate_vm()
236 for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) { in first_copy_engine()
237 engine = gt->engine_class[COPY_ENGINE_CLASS][i]; in first_copy_engine()
254 return ERR_PTR(-ENODEV); in pinned_context()
277 m->context = ce; in intel_migrate_init()
293 for (i = 0; i < ARRAY_SIZE(gt->engine_class[COPY_ENGINE_CLASS]); i++) { in __migrate_engines()
294 engine = gt->engine_class[COPY_ENGINE_CLASS][i]; in __migrate_engines()
312 * balancing of the virtual-engine. in intel_migrate_create_context()
314 ce = __migrate_engines(m->context->engine->gt); in intel_migrate_create_context()
318 ce->ring = NULL; in intel_migrate_create_context()
319 ce->ring_size = SZ_256K; in intel_migrate_create_context()
321 i915_vm_put(ce->vm); in intel_migrate_create_context()
322 ce->vm = i915_vm_get(m->context->vm); in intel_migrate_create_context()
336 u32 *cs; in emit_no_arbitration() local
338 cs = intel_ring_begin(rq, 2); in emit_no_arbitration()
339 if (IS_ERR(cs)) in emit_no_arbitration()
340 return PTR_ERR(cs); in emit_no_arbitration()
343 *cs++ = MI_ARB_ON_OFF; in emit_no_arbitration()
344 *cs++ = MI_NOOP; in emit_no_arbitration()
345 intel_ring_advance(rq, cs); in emit_no_arbitration()
352 struct intel_ring *ring = rq->ring; in max_pte_pkt_size()
354 pkt = min_t(int, pkt, (ring->space - rq->reserved_space) / sizeof(u32) + 5); in max_pte_pkt_size()
355 pkt = min_t(int, pkt, (ring->size - ring->emit) / sizeof(u32) + 5); in max_pte_pkt_size()
369 bool has_64K_pages = HAS_64K_PAGES(rq->i915); in emit_pte()
370 const u64 encode = rq->context->vm->pte_encode(0, pat_index, in emit_pte()
372 struct intel_ring *ring = rq->ring; in emit_pte()
376 u32 *hdr, *cs; in emit_pte() local
378 GEM_BUG_ON(GRAPHICS_VER(rq->i915) < 8); in emit_pte()
389 offset += 3 * CHUNK_SZ; in emit_pte()
401 offset += (u64)rq->engine->instance << 32; in emit_pte()
403 cs = intel_ring_begin(rq, I915_EMIT_PTE_NUM_DWORDS); in emit_pte()
404 if (IS_ERR(cs)) in emit_pte()
405 return PTR_ERR(cs); in emit_pte()
410 hdr = cs; in emit_pte()
411 *cs++ = MI_STORE_DATA_IMM | REG_BIT(21); /* as qword elements */ in emit_pte()
412 *cs++ = lower_32_bits(offset); in emit_pte()
413 *cs++ = upper_32_bits(offset); in emit_pte()
416 if (cs - hdr >= pkt) { in emit_pte()
419 *hdr += cs - hdr - 2; in emit_pte()
420 *cs++ = MI_NOOP; in emit_pte()
422 ring->emit = (void *)cs - ring->vaddr; in emit_pte()
423 intel_ring_advance(rq, cs); in emit_pte()
426 cs = intel_ring_begin(rq, I915_EMIT_PTE_NUM_DWORDS); in emit_pte()
427 if (IS_ERR(cs)) in emit_pte()
428 return PTR_ERR(cs); in emit_pte()
435 dword_rem = SZ_2M - (total & (SZ_2M - 1)); in emit_pte()
443 hdr = cs; in emit_pte()
444 *cs++ = MI_STORE_DATA_IMM | REG_BIT(21); in emit_pte()
445 *cs++ = lower_32_bits(offset); in emit_pte()
446 *cs++ = upper_32_bits(offset); in emit_pte()
449 GEM_BUG_ON(!IS_ALIGNED(it->dma, page_size)); in emit_pte()
451 *cs++ = lower_32_bits(encode | it->dma); in emit_pte()
452 *cs++ = upper_32_bits(encode | it->dma); in emit_pte()
457 it->dma += page_size; in emit_pte()
458 if (it->dma >= it->max) { in emit_pte()
459 it->sg = __sg_next(it->sg); in emit_pte()
460 if (!it->sg || sg_dma_len(it->sg) == 0) in emit_pte()
463 it->dma = sg_dma_address(it->sg); in emit_pte()
464 it->max = it->dma + sg_dma_len(it->sg); in emit_pte()
468 *hdr += cs - hdr - 2; in emit_pte()
469 *cs++ = MI_NOOP; in emit_pte()
471 ring->emit = (void *)cs - ring->vaddr; in emit_pte()
472 intel_ring_advance(rq, cs); in emit_pte()
485 return height % 4 == 3 && height <= 8; in wa_1209644611_applies()
489 * DOC: Flat-CCS - Memory compression for Local memory
491 * On Xe-HP and later devices, we use dedicated compression control state (CCS)
492 * stored in local memory for each surface, to support the 3D and media
504 * I915 supports Flat-CCS on lmem only objects. When an objects has smem in
506 * content into smem. If the lmem object is Flat-CCS compressed by userspace,
508 * for such decompression. Hence I915 supports Flat-CCS only on lmem only objects.
510 * When we exhaust the lmem, Flat-CCS capable objects' lmem backing memory can
512 * it can be potentially swapped-out at a later point, if required.
515 * and potentially performing any required swap-in.
518 * {lmem, smem}, objects are treated as non Flat-CCS capable objects.
534 struct drm_i915_private *i915 = rq->i915; in emit_copy_ccs()
535 int mocs = rq->engine->gt->mocs.uc_index << 1; in emit_copy_ccs()
537 u32 *cs; in emit_copy_ccs() local
539 cs = intel_ring_begin(rq, 12); in emit_copy_ccs()
540 if (IS_ERR(cs)) in emit_copy_ccs()
541 return PTR_ERR(cs); in emit_copy_ccs()
546 cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS); in emit_copy_ccs()
562 *cs++ = XY_CTRL_SURF_COPY_BLT | in emit_copy_ccs()
565 ((num_ccs_blks - 1) & CCS_SIZE_MASK) << CCS_SIZE_SHIFT; in emit_copy_ccs()
566 *cs++ = src_offset; in emit_copy_ccs()
567 *cs++ = rq->engine->instance | in emit_copy_ccs()
569 *cs++ = dst_offset; in emit_copy_ccs()
570 *cs++ = rq->engine->instance | in emit_copy_ccs()
573 cs = i915_flush_dw(cs, MI_FLUSH_DW_LLC | MI_FLUSH_DW_CCS); in emit_copy_ccs()
574 *cs++ = MI_NOOP; in emit_copy_ccs()
576 intel_ring_advance(rq, cs); in emit_copy_ccs()
584 const int ver = GRAPHICS_VER(rq->i915); in emit_copy()
585 u32 instance = rq->engine->instance; in emit_copy()
586 u32 *cs; in emit_copy() local
588 cs = intel_ring_begin(rq, ver >= 8 ? 10 : 6); in emit_copy()
589 if (IS_ERR(cs)) in emit_copy()
590 return PTR_ERR(cs); in emit_copy()
593 *cs++ = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2); in emit_copy()
594 *cs++ = BLT_DEPTH_32 | PAGE_SIZE; in emit_copy()
595 *cs++ = 0; in emit_copy()
596 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; in emit_copy()
597 *cs++ = dst_offset; in emit_copy()
598 *cs++ = instance; in emit_copy()
599 *cs++ = 0; in emit_copy()
600 *cs++ = PAGE_SIZE; in emit_copy()
601 *cs++ = src_offset; in emit_copy()
602 *cs++ = instance; in emit_copy()
604 *cs++ = XY_SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (10 - 2); in emit_copy()
605 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; in emit_copy()
606 *cs++ = 0; in emit_copy()
607 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; in emit_copy()
608 *cs++ = dst_offset; in emit_copy()
609 *cs++ = instance; in emit_copy()
610 *cs++ = 0; in emit_copy()
611 *cs++ = PAGE_SIZE; in emit_copy()
612 *cs++ = src_offset; in emit_copy()
613 *cs++ = instance; in emit_copy()
616 *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); in emit_copy()
617 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | PAGE_SIZE; in emit_copy()
618 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE; in emit_copy()
619 *cs++ = dst_offset; in emit_copy()
620 *cs++ = PAGE_SIZE; in emit_copy()
621 *cs++ = src_offset; in emit_copy()
624 intel_ring_advance(rq, cs); in emit_copy()
647 * will be taken for the blt. in Flat-ccs supported in calculate_chunk_sz()
662 GEM_BUG_ON(!it->sg || !sg_dma_len(it->sg)); in get_ccs_sg_sgt()
663 len = it->max - it->dma; in get_ccs_sg_sgt()
665 it->dma += bytes_to_cpy; in get_ccs_sg_sgt()
669 bytes_to_cpy -= len; in get_ccs_sg_sgt()
671 it->sg = __sg_next(it->sg); in get_ccs_sg_sgt()
672 it->dma = sg_dma_address(it->sg); in get_ccs_sg_sgt()
673 it->max = it->dma + sg_dma_len(it->sg); in get_ccs_sg_sgt()
689 struct drm_i915_private *i915 = ce->engine->i915; in intel_context_migrate_copy()
699 GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm); in intel_context_migrate_copy()
700 GEM_BUG_ON(IS_DGFX(ce->engine->i915) && (!src_is_lmem && !dst_is_lmem)); in intel_context_migrate_copy()
703 GEM_BUG_ON(ce->ring->size < SZ_64K); in intel_context_migrate_copy()
728 * TO-DO: Want to move the size mismatch check to a WARN_ON, in intel_context_migrate_copy()
729 * but still we have some requests of smem->lmem with same size. in intel_context_migrate_copy()
741 if (HAS_64K_PAGES(ce->engine->i915)) { in intel_context_migrate_copy()
764 if (rq->engine->emit_init_breadcrumb) { in intel_context_migrate_copy()
765 err = rq->engine->emit_init_breadcrumb(rq); in intel_context_migrate_copy()
784 err = -EINVAL; in intel_context_migrate_copy()
797 err = -EINVAL; in intel_context_migrate_copy()
801 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); in intel_context_migrate_copy()
809 bytes_to_cpy -= len; in intel_context_migrate_copy()
814 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); in intel_context_migrate_copy()
825 err = -EINVAL; in intel_context_migrate_copy()
829 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); in intel_context_migrate_copy()
838 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); in intel_context_migrate_copy()
841 ccs_bytes_to_cpy -= ccs_sz; in intel_context_migrate_copy()
843 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); in intel_context_migrate_copy()
850 * be doing an lmem -> lmem transfer, and so in intel_context_migrate_copy()
857 * need to copy the CCS state as-is. in intel_context_migrate_copy()
879 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); in intel_context_migrate_copy()
884 /* Arbitration is re-enabled between requests. */ in intel_context_migrate_copy()
906 err = -EINVAL; in intel_context_migrate_copy()
920 struct drm_i915_private *i915 = rq->i915; in emit_clear()
921 int mocs = rq->engine->gt->mocs.uc_index << 1; in emit_clear()
924 u32 *cs; in emit_clear() local
935 cs = intel_ring_begin(rq, ring_sz); in emit_clear()
936 if (IS_ERR(cs)) in emit_clear()
937 return PTR_ERR(cs); in emit_clear()
940 *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | in emit_clear()
941 (XY_FAST_COLOR_BLT_DW - 2); in emit_clear()
942 *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) | in emit_clear()
943 (PAGE_SIZE - 1); in emit_clear()
944 *cs++ = 0; in emit_clear()
945 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; in emit_clear()
946 *cs++ = offset; in emit_clear()
947 *cs++ = rq->engine->instance; in emit_clear()
948 *cs++ = !is_lmem << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; in emit_clear()
950 *cs++ = value; in emit_clear()
951 *cs++ = 0; in emit_clear()
952 *cs++ = 0; in emit_clear()
953 *cs++ = 0; in emit_clear()
955 *cs++ = 0; in emit_clear()
956 *cs++ = 0; in emit_clear()
958 *cs++ = 0; in emit_clear()
959 *cs++ = 0; in emit_clear()
960 *cs++ = 0; in emit_clear()
962 *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (7 - 2); in emit_clear()
963 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; in emit_clear()
964 *cs++ = 0; in emit_clear()
965 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; in emit_clear()
966 *cs++ = offset; in emit_clear()
967 *cs++ = rq->engine->instance; in emit_clear()
968 *cs++ = value; in emit_clear()
969 *cs++ = MI_NOOP; in emit_clear()
971 *cs++ = XY_COLOR_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); in emit_clear()
972 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | PAGE_SIZE; in emit_clear()
973 *cs++ = 0; in emit_clear()
974 *cs++ = size >> PAGE_SHIFT << 16 | PAGE_SIZE / 4; in emit_clear()
975 *cs++ = offset; in emit_clear()
976 *cs++ = value; in emit_clear()
979 intel_ring_advance(rq, cs); in emit_clear()
992 struct drm_i915_private *i915 = ce->engine->i915; in intel_context_migrate_clear()
998 GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm); in intel_context_migrate_clear()
1001 GEM_BUG_ON(ce->ring->size < SZ_64K); in intel_context_migrate_clear()
1021 if (rq->engine->emit_init_breadcrumb) { in intel_context_migrate_clear()
1022 err = rq->engine->emit_init_breadcrumb(rq); in intel_context_migrate_clear()
1041 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); in intel_context_migrate_clear()
1060 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE); in intel_context_migrate_clear()
1062 /* Arbitration is re-enabled between requests. */ in intel_context_migrate_clear()
1093 if (!m->context) in intel_migrate_copy()
1094 return -ENODEV; in intel_migrate_copy()
1098 ce = intel_context_get(m->context); in intel_migrate_copy()
1130 if (!m->context) in intel_migrate_clear()
1131 return -ENODEV; in intel_migrate_clear()
1135 ce = intel_context_get(m->context); in intel_migrate_clear()
1155 ce = fetch_and_zero(&m->context); in intel_migrate_fini()