Lines Matching +full:1 +full:eb
50 FORCE_CPU_RELOC = 1,
106 * 1. Add a command to load the HW context. For Logical Ring Contexts, i.e.
133 * 1. Validation - Ensure all the pointers, handles and flags are valid.
259 struct i915_request *requests[MAX_ENGINE_INSTANCE + 1];
261 struct eb_vma *batches[MAX_ENGINE_INSTANCE + 1];
264 /** used for excl fence in dma_resv objects when > 1 BB submitted */
291 bool use_64bit_reloc : 1;
292 bool has_llc : 1;
293 bool has_fence : 1;
294 bool needs_unfenced : 1;
300 u64 batch_len[MAX_ENGINE_INSTANCE + 1];
316 struct i915_capture_list *capture_lists[MAX_ENGINE_INSTANCE + 1];
320 static int eb_parse(struct i915_execbuffer *eb);
321 static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle);
322 static void eb_unpin_engine(struct i915_execbuffer *eb);
323 static void eb_capture_release(struct i915_execbuffer *eb);
325 static bool eb_use_cmdparser(const struct i915_execbuffer *eb) in eb_use_cmdparser() argument
327 return intel_engine_requires_cmd_parser(eb->context->engine) || in eb_use_cmdparser()
328 (intel_engine_using_cmd_parser(eb->context->engine) && in eb_use_cmdparser()
329 eb->args->batch_len); in eb_use_cmdparser()
332 static int eb_create(struct i915_execbuffer *eb) in eb_create() argument
334 if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) { in eb_create()
335 unsigned int size = 1 + ilog2(eb->buffer_count); in eb_create()
338 * Without a 1:1 association between relocation handles and in eb_create()
341 * first with 1:1 associative hash and scaling back until in eb_create()
358 if (size > 1) in eb_create()
361 eb->buckets = kzalloc(sizeof(struct hlist_head) << size, in eb_create()
363 if (eb->buckets) in eb_create()
370 eb->lut_size = size; in eb_create()
372 eb->lut_size = -eb->buffer_count; in eb_create()
438 eb_pin_vma(struct i915_execbuffer *eb, in eb_pin_vma() argument
456 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags); in eb_pin_vma()
465 err = i915_vma_pin_ww(vma, &eb->ww, in eb_pin_vma()
500 eb_validate_vma(struct i915_execbuffer *eb, in eb_validate_vma() argument
508 GRAPHICS_VER(eb->i915) >= 12 && !IS_TIGERLAKE(eb->i915)) in eb_validate_vma()
511 if (unlikely(entry->flags & eb->invalid_flags)) in eb_validate_vma()
540 if (!eb->reloc_cache.has_fence) { in eb_validate_vma()
544 eb->reloc_cache.needs_unfenced) && in eb_validate_vma()
553 is_batch_buffer(struct i915_execbuffer *eb, unsigned int buffer_idx) in is_batch_buffer() argument
555 return eb->args->flags & I915_EXEC_BATCH_FIRST ? in is_batch_buffer()
556 buffer_idx < eb->num_batches : in is_batch_buffer()
557 buffer_idx >= eb->args->buffer_count - eb->num_batches; in is_batch_buffer()
561 eb_add_vma(struct i915_execbuffer *eb, in eb_add_vma() argument
566 struct drm_i915_private *i915 = eb->i915; in eb_add_vma()
567 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; in eb_add_vma()
568 struct eb_vma *ev = &eb->vma[i]; in eb_add_vma()
574 if (eb->lut_size > 0) { in eb_add_vma()
577 &eb->buckets[hash_32(entry->handle, in eb_add_vma()
578 eb->lut_size)]); in eb_add_vma()
582 list_add_tail(&ev->reloc_link, &eb->relocs); in eb_add_vma()
593 if (is_batch_buffer(eb, i)) { in eb_add_vma()
597 if (eb->reloc_cache.has_fence) in eb_add_vma()
600 eb->batches[*current_batch] = ev; in eb_add_vma()
609 eb->batch_start_offset, in eb_add_vma()
610 eb->args->batch_len, in eb_add_vma()
616 if (eb->args->batch_len == 0) in eb_add_vma()
617 eb->batch_len[*current_batch] = ev->vma->size - in eb_add_vma()
618 eb->batch_start_offset; in eb_add_vma()
620 eb->batch_len[*current_batch] = eb->args->batch_len; in eb_add_vma()
621 if (unlikely(eb->batch_len[*current_batch] == 0)) { /* impossible! */ in eb_add_vma()
655 static int eb_reserve_vma(struct i915_execbuffer *eb, in eb_reserve_vma() argument
670 err = i915_vma_pin_ww(vma, &eb->ww, in eb_reserve_vma()
678 eb->args->flags |= __EXEC_HAS_RELOC; in eb_reserve_vma()
696 static bool eb_unbind(struct i915_execbuffer *eb, bool force) in eb_unbind() argument
698 const unsigned int count = eb->buffer_count; in eb_unbind()
704 INIT_LIST_HEAD(&eb->unbound); in eb_unbind()
708 struct eb_vma *ev = &eb->vma[i]; in eb_unbind()
720 list_add(&ev->bind_link, &eb->unbound); in eb_unbind()
723 list_add_tail(&ev->bind_link, &eb->unbound); in eb_unbind()
731 list_splice_tail(&last, &eb->unbound); in eb_unbind()
735 static int eb_reserve(struct i915_execbuffer *eb) in eb_reserve() argument
751 * 1. Reorder the buffers, where objects with the most restrictive in eb_reserve()
778 if (pass >= 1) in eb_reserve()
779 eb_unbind(eb, pass >= 2); in eb_reserve()
782 err = mutex_lock_interruptible(&eb->context->vm->mutex); in eb_reserve()
784 err = i915_gem_evict_vm(eb->context->vm, &eb->ww, NULL); in eb_reserve()
785 mutex_unlock(&eb->context->vm->mutex); in eb_reserve()
793 err = mutex_lock_interruptible(&eb->context->vm->mutex); in eb_reserve()
797 err = i915_gem_evict_vm(eb->context->vm, &eb->ww, &busy_bo); in eb_reserve()
798 mutex_unlock(&eb->context->vm->mutex); in eb_reserve()
800 err = i915_gem_object_lock(busy_bo, &eb->ww); in eb_reserve()
810 list_for_each_entry(ev, &eb->unbound, bind_link) { in eb_reserve()
811 err = eb_reserve_vma(eb, ev, pin_flags); in eb_reserve()
823 static int eb_select_context(struct i915_execbuffer *eb) in eb_select_context() argument
827 ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1); in eb_select_context()
831 eb->gem_context = ctx; in eb_select_context()
833 eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT; in eb_select_context()
838 static int __eb_add_lut(struct i915_execbuffer *eb, in __eb_add_lut() argument
841 struct i915_gem_context *ctx = eb->gem_context; in __eb_add_lut()
866 if (idr_find(&eb->file->object_idr, handle) == obj) { in __eb_add_lut()
888 static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) in eb_lookup_vma() argument
890 struct i915_address_space *vm = eb->context->vm; in eb_lookup_vma()
898 vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle); in eb_lookup_vma()
907 obj = i915_gem_object_lookup(eb->file, handle); in eb_lookup_vma()
918 if (i915_gem_context_uses_protected_content(eb->gem_context) && in eb_lookup_vma()
933 err = __eb_add_lut(eb, handle, vma); in eb_lookup_vma()
940 } while (1); in eb_lookup_vma()
943 static int eb_lookup_vmas(struct i915_execbuffer *eb) in eb_lookup_vmas() argument
948 INIT_LIST_HEAD(&eb->relocs); in eb_lookup_vmas()
950 for (i = 0; i < eb->buffer_count; i++) { in eb_lookup_vmas()
953 vma = eb_lookup_vma(eb, eb->exec[i].handle); in eb_lookup_vmas()
959 err = eb_validate_vma(eb, &eb->exec[i], vma); in eb_lookup_vmas()
965 err = eb_add_vma(eb, ¤t_batch, i, vma); in eb_lookup_vmas()
974 eb->vma[i].flags |= __EXEC_OBJECT_USERPTR_INIT; in eb_lookup_vmas()
975 eb->args->flags |= __EXEC_USERPTR_USED; in eb_lookup_vmas()
982 static int eb_lock_vmas(struct i915_execbuffer *eb) in eb_lock_vmas() argument
987 for (i = 0; i < eb->buffer_count; i++) { in eb_lock_vmas()
988 struct eb_vma *ev = &eb->vma[i]; in eb_lock_vmas()
991 err = i915_gem_object_lock(vma->obj, &eb->ww); in eb_lock_vmas()
999 static int eb_validate_vmas(struct i915_execbuffer *eb) in eb_validate_vmas() argument
1004 INIT_LIST_HEAD(&eb->unbound); in eb_validate_vmas()
1006 err = eb_lock_vmas(eb); in eb_validate_vmas()
1010 for (i = 0; i < eb->buffer_count; i++) { in eb_validate_vmas()
1011 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; in eb_validate_vmas()
1012 struct eb_vma *ev = &eb->vma[i]; in eb_validate_vmas()
1015 err = eb_pin_vma(eb, entry, ev); in eb_validate_vmas()
1022 eb->args->flags |= __EXEC_HAS_RELOC; in eb_validate_vmas()
1027 list_add_tail(&ev->bind_link, &eb->unbound); in eb_validate_vmas()
1036 err = dma_resv_reserve_fences(vma->obj->base.resv, eb->num_batches); in eb_validate_vmas()
1041 eb_vma_misplaced(&eb->exec[i], vma, ev->flags)); in eb_validate_vmas()
1044 if (!list_empty(&eb->unbound)) in eb_validate_vmas()
1045 return eb_reserve(eb); in eb_validate_vmas()
1051 eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) in eb_get_vma() argument
1053 if (eb->lut_size < 0) { in eb_get_vma()
1054 if (handle >= -eb->lut_size) in eb_get_vma()
1056 return &eb->vma[handle]; in eb_get_vma()
1061 head = &eb->buckets[hash_32(handle, eb->lut_size)]; in eb_get_vma()
1070 static void eb_release_vmas(struct i915_execbuffer *eb, bool final) in eb_release_vmas() argument
1072 const unsigned int count = eb->buffer_count; in eb_release_vmas()
1076 struct eb_vma *ev = &eb->vma[i]; in eb_release_vmas()
1088 eb_capture_release(eb); in eb_release_vmas()
1089 eb_unpin_engine(eb); in eb_release_vmas()
1092 static void eb_destroy(const struct i915_execbuffer *eb) in eb_destroy() argument
1094 if (eb->lut_size > 0) in eb_destroy()
1095 kfree(eb->buckets); in eb_destroy()
1108 cache->page = -1; in reloc_cache_init()
1179 static void reloc_cache_reset(struct reloc_cache *cache, struct i915_execbuffer *eb) in reloc_cache_reset() argument
1214 cache->page = -1; in reloc_cache_reset()
1255 struct i915_execbuffer *eb, in reloc_iomap() argument
1259 struct reloc_cache *cache = &eb->reloc_cache; in reloc_iomap()
1290 vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0, in reloc_iomap()
1337 struct i915_execbuffer *eb, in reloc_vaddr() argument
1340 struct reloc_cache *cache = &eb->reloc_cache; in reloc_vaddr()
1348 vaddr = reloc_iomap(vma, eb, page); in reloc_vaddr()
1381 struct i915_execbuffer *eb, in relocate_entry() argument
1386 bool wide = eb->reloc_cache.use_64bit_reloc; in relocate_entry()
1390 vaddr = reloc_vaddr(vma, eb, in relocate_entry()
1398 eb->reloc_cache.vaddr); in relocate_entry()
1411 eb_relocate_entry(struct i915_execbuffer *eb, in eb_relocate_entry() argument
1415 struct drm_i915_private *i915 = eb->i915; in eb_relocate_entry()
1420 target = eb_get_vma(eb, reloc->target_handle); in eb_relocate_entry()
1425 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) { in eb_relocate_entry()
1457 GRAPHICS_VER(eb->i915) == 6 && in eb_relocate_entry()
1461 reloc_cache_unmap(&eb->reloc_cache); in eb_relocate_entry()
1467 reloc_cache_remap(&eb->reloc_cache, ev->vma->obj); in eb_relocate_entry()
1483 ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) { in eb_relocate_entry()
1510 return relocate_entry(ev->vma, reloc, eb, target->vma); in eb_relocate_entry()
1513 static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) in eb_relocate_vma() argument
1557 u64 offset = eb_relocate_entry(eb, ev, r); in eb_relocate_vma()
1593 reloc_cache_reset(&eb->reloc_cache, eb); in eb_relocate_vma()
1598 eb_relocate_vma_slow(struct i915_execbuffer *eb, struct eb_vma *ev) in eb_relocate_vma_slow() argument
1607 u64 offset = eb_relocate_entry(eb, ev, &relocs[i]); in eb_relocate_vma_slow()
1616 reloc_cache_reset(&eb->reloc_cache, eb); in eb_relocate_vma_slow()
1644 return __get_user(c, end - 1); in check_relocations()
1647 static int eb_copy_relocations(const struct i915_execbuffer *eb) in eb_copy_relocations() argument
1650 const unsigned int count = eb->buffer_count; in eb_copy_relocations()
1655 const unsigned int nreloc = eb->exec[i].relocation_count; in eb_copy_relocations()
1663 err = check_relocations(&eb->exec[i]); in eb_copy_relocations()
1667 urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr); in eb_copy_relocations()
1670 relocs = kvmalloc_array(1, size, GFP_KERNEL); in eb_copy_relocations()
1704 unsafe_put_user(-1, in eb_copy_relocations()
1709 eb->exec[i].relocs_ptr = (uintptr_t)relocs; in eb_copy_relocations()
1721 relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr); in eb_copy_relocations()
1722 if (eb->exec[i].relocation_count) in eb_copy_relocations()
1728 static int eb_prefault_relocations(const struct i915_execbuffer *eb) in eb_prefault_relocations() argument
1730 const unsigned int count = eb->buffer_count; in eb_prefault_relocations()
1736 err = check_relocations(&eb->exec[i]); in eb_prefault_relocations()
1744 static int eb_reinit_userptr(struct i915_execbuffer *eb) in eb_reinit_userptr() argument
1746 const unsigned int count = eb->buffer_count; in eb_reinit_userptr()
1750 if (likely(!(eb->args->flags & __EXEC_USERPTR_USED))) in eb_reinit_userptr()
1754 struct eb_vma *ev = &eb->vma[i]; in eb_reinit_userptr()
1769 static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb) in eb_relocate_parse_slow() argument
1782 eb_release_vmas(eb, false); in eb_relocate_parse_slow()
1783 i915_gem_ww_ctx_fini(&eb->ww); in eb_relocate_parse_slow()
1788 * 1 - we try to just prefault all the user relocation entries and in eb_relocate_parse_slow()
1799 err = eb_prefault_relocations(eb); in eb_relocate_parse_slow()
1801 err = eb_copy_relocations(eb); in eb_relocate_parse_slow()
1809 err = eb_reinit_userptr(eb); in eb_relocate_parse_slow()
1811 i915_gem_ww_ctx_init(&eb->ww, true); in eb_relocate_parse_slow()
1817 err = eb_pin_engine(eb, false); in eb_relocate_parse_slow()
1821 err = eb_validate_vmas(eb); in eb_relocate_parse_slow()
1825 GEM_BUG_ON(!eb->batches[0]); in eb_relocate_parse_slow()
1827 list_for_each_entry(ev, &eb->relocs, reloc_link) { in eb_relocate_parse_slow()
1829 err = eb_relocate_vma(eb, ev); in eb_relocate_parse_slow()
1833 err = eb_relocate_vma_slow(eb, ev); in eb_relocate_parse_slow()
1849 err = eb_parse(eb); in eb_relocate_parse_slow()
1862 eb_release_vmas(eb, false); in eb_relocate_parse_slow()
1863 err = i915_gem_ww_ctx_backoff(&eb->ww); in eb_relocate_parse_slow()
1873 const unsigned int count = eb->buffer_count; in eb_relocate_parse_slow()
1878 &eb->exec[i]; in eb_relocate_parse_slow()
1892 static int eb_relocate_parse(struct i915_execbuffer *eb) in eb_relocate_parse() argument
1898 err = eb_pin_engine(eb, throttle); in eb_relocate_parse()
1909 err = eb_validate_vmas(eb); in eb_relocate_parse()
1916 if (eb->args->flags & __EXEC_HAS_RELOC) { in eb_relocate_parse()
1919 list_for_each_entry(ev, &eb->relocs, reloc_link) { in eb_relocate_parse()
1920 err = eb_relocate_vma(eb, ev); in eb_relocate_parse()
1932 err = eb_parse(eb); in eb_relocate_parse()
1936 eb_release_vmas(eb, false); in eb_relocate_parse()
1937 err = i915_gem_ww_ctx_backoff(&eb->ww); in eb_relocate_parse()
1945 err = eb_relocate_parse_slow(eb); in eb_relocate_parse()
1954 eb->args->flags &= ~__EXEC_HAS_RELOC; in eb_relocate_parse()
1972 for ((_i) = (_eb)->num_batches - 1; (_i) >= 0; --(_i))
1975 eb_find_first_request_added(struct i915_execbuffer *eb) in eb_find_first_request_added() argument
1979 for_each_batch_add_order(eb, i) in eb_find_first_request_added()
1980 if (eb->requests[i]) in eb_find_first_request_added()
1981 return eb->requests[i]; in eb_find_first_request_added()
1991 static int eb_capture_stage(struct i915_execbuffer *eb) in eb_capture_stage() argument
1993 const unsigned int count = eb->buffer_count; in eb_capture_stage()
1997 struct eb_vma *ev = &eb->vma[i]; in eb_capture_stage()
2004 if (i915_gem_context_is_recoverable(eb->gem_context) && in eb_capture_stage()
2005 (IS_DGFX(eb->i915) || GRAPHICS_VER_FULL(eb->i915) > IP_VER(12, 0))) in eb_capture_stage()
2008 for_each_batch_create_order(eb, j) { in eb_capture_stage()
2015 capture->next = eb->capture_lists[j]; in eb_capture_stage()
2017 eb->capture_lists[j] = capture; in eb_capture_stage()
2025 static void eb_capture_commit(struct i915_execbuffer *eb) in eb_capture_commit() argument
2029 for_each_batch_create_order(eb, j) { in eb_capture_commit()
2030 struct i915_request *rq = eb->requests[j]; in eb_capture_commit()
2035 rq->capture_list = eb->capture_lists[j]; in eb_capture_commit()
2036 eb->capture_lists[j] = NULL; in eb_capture_commit()
2044 static void eb_capture_release(struct i915_execbuffer *eb) in eb_capture_release() argument
2048 for_each_batch_create_order(eb, j) { in eb_capture_release()
2049 if (eb->capture_lists[j]) { in eb_capture_release()
2050 i915_request_free_capture_list(eb->capture_lists[j]); in eb_capture_release()
2051 eb->capture_lists[j] = NULL; in eb_capture_release()
2056 static void eb_capture_list_clear(struct i915_execbuffer *eb) in eb_capture_list_clear() argument
2058 memset(eb->capture_lists, 0, sizeof(eb->capture_lists)); in eb_capture_list_clear()
2063 static int eb_capture_stage(struct i915_execbuffer *eb) in eb_capture_stage() argument
2068 static void eb_capture_commit(struct i915_execbuffer *eb) in eb_capture_commit() argument
2072 static void eb_capture_release(struct i915_execbuffer *eb) in eb_capture_release() argument
2076 static void eb_capture_list_clear(struct i915_execbuffer *eb) in eb_capture_list_clear() argument
2082 static int eb_move_to_gpu(struct i915_execbuffer *eb) in eb_move_to_gpu() argument
2084 const unsigned int count = eb->buffer_count; in eb_move_to_gpu()
2089 struct eb_vma *ev = &eb->vma[i]; in eb_move_to_gpu()
2113 * 1. Rip out the async flush code. in eb_move_to_gpu()
2127 (eb_find_first_request_added(eb), obj, in eb_move_to_gpu()
2131 for_each_batch_add_order(eb, j) { in eb_move_to_gpu()
2134 if (!eb->requests[j]) in eb_move_to_gpu()
2137 err = _i915_vma_move_to_active(vma, eb->requests[j], in eb_move_to_gpu()
2139 eb->composite_fence ? in eb_move_to_gpu()
2140 eb->composite_fence : in eb_move_to_gpu()
2141 &eb->requests[j]->fence, in eb_move_to_gpu()
2148 if (!err && (eb->args->flags & __EXEC_USERPTR_USED)) { in eb_move_to_gpu()
2150 struct eb_vma *ev = &eb->vma[i]; in eb_move_to_gpu()
2167 intel_gt_chipset_flush(eb->gt); in eb_move_to_gpu()
2168 eb_capture_commit(eb); in eb_move_to_gpu()
2173 for_each_batch_create_order(eb, j) { in eb_move_to_gpu()
2174 if (!eb->requests[j]) in eb_move_to_gpu()
2177 i915_request_set_error_once(eb->requests[j], err); in eb_move_to_gpu()
2234 shadow_batch_pin(struct i915_execbuffer *eb, in shadow_batch_pin() argument
2246 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags | PIN_VALIDATE); in shadow_batch_pin()
2253 static struct i915_vma *eb_dispatch_secure(struct i915_execbuffer *eb, struct i915_vma *vma) in eb_dispatch_secure() argument
2259 if (eb->batch_flags & I915_DISPATCH_SECURE) in eb_dispatch_secure()
2260 return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, PIN_VALIDATE); in eb_dispatch_secure()
2265 static int eb_parse(struct i915_execbuffer *eb) in eb_parse() argument
2267 struct drm_i915_private *i915 = eb->i915; in eb_parse()
2268 struct intel_gt_buffer_pool_node *pool = eb->batch_pool; in eb_parse()
2273 if (!eb_use_cmdparser(eb)) { in eb_parse()
2274 batch = eb_dispatch_secure(eb, eb->batches[0]->vma); in eb_parse()
2281 if (intel_context_is_parallel(eb->context)) in eb_parse()
2284 len = eb->batch_len[0]; in eb_parse()
2285 if (!CMDPARSER_USES_GGTT(eb->i915)) { in eb_parse()
2290 if (!eb->context->vm->has_read_only) { in eb_parse()
2298 if (unlikely(len < eb->batch_len[0])) /* last paranoid check of overflow */ in eb_parse()
2302 pool = intel_gt_get_buffer_pool(eb->gt, len, in eb_parse()
2306 eb->batch_pool = pool; in eb_parse()
2309 err = i915_gem_object_lock(pool->obj, &eb->ww); in eb_parse()
2313 shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER); in eb_parse()
2322 if (CMDPARSER_USES_GGTT(eb->i915)) { in eb_parse()
2325 shadow = shadow_batch_pin(eb, pool->obj, in eb_parse()
2326 &eb->gt->ggtt->vm, in eb_parse()
2333 eb->batch_flags |= I915_DISPATCH_SECURE; in eb_parse()
2336 batch = eb_dispatch_secure(eb, shadow); in eb_parse()
2340 err = dma_resv_reserve_fences(shadow->obj->base.resv, 1); in eb_parse()
2344 err = intel_engine_cmd_parser(eb->context->engine, in eb_parse()
2345 eb->batches[0]->vma, in eb_parse()
2346 eb->batch_start_offset, in eb_parse()
2347 eb->batch_len[0], in eb_parse()
2352 eb->batches[0] = &eb->vma[eb->buffer_count++]; in eb_parse()
2353 eb->batches[0]->vma = i915_vma_get(shadow); in eb_parse()
2354 eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN; in eb_parse()
2356 eb->trampoline = trampoline; in eb_parse()
2357 eb->batch_start_offset = 0; in eb_parse()
2361 if (intel_context_is_parallel(eb->context)) in eb_parse()
2364 eb->batches[0] = &eb->vma[eb->buffer_count++]; in eb_parse()
2365 eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN; in eb_parse()
2366 eb->batches[0]->vma = i915_vma_get(batch); in eb_parse()
2371 static int eb_request_submit(struct i915_execbuffer *eb, in eb_request_submit() argument
2381 if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) { in eb_request_submit()
2401 eb->batch_start_offset, in eb_request_submit()
2403 eb->batch_flags); in eb_request_submit()
2407 if (eb->trampoline) { in eb_request_submit()
2409 GEM_BUG_ON(eb->batch_start_offset); in eb_request_submit()
2411 i915_vma_offset(eb->trampoline) + in eb_request_submit()
2420 static int eb_submit(struct i915_execbuffer *eb) in eb_submit() argument
2425 err = eb_move_to_gpu(eb); in eb_submit()
2427 for_each_batch_create_order(eb, i) { in eb_submit()
2428 if (!eb->requests[i]) in eb_submit()
2431 trace_i915_request_queue(eb->requests[i], eb->batch_flags); in eb_submit()
2433 err = eb_request_submit(eb, eb->requests[i], in eb_submit()
2434 eb->batches[i]->vma, in eb_submit()
2435 eb->batch_len[i]); in eb_submit()
2467 static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel_context *ce) in eb_throttle() argument
2501 static int eb_pin_timeline(struct i915_execbuffer *eb, struct intel_context *ce, in eb_pin_timeline() argument
2521 rq = eb_throttle(eb, ce); in eb_pin_timeline()
2525 bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; in eb_pin_timeline()
2552 static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle) in eb_pin_engine() argument
2554 struct intel_context *ce = eb->context, *child; in eb_pin_engine()
2558 GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED); in eb_pin_engine()
2568 err = intel_context_pin_ww(ce, &eb->ww); in eb_pin_engine()
2572 err = intel_context_pin_ww(child, &eb->ww); in eb_pin_engine()
2577 err = eb_pin_timeline(eb, child, throttle); in eb_pin_engine()
2582 err = eb_pin_timeline(eb, ce, throttle); in eb_pin_engine()
2586 eb->args->flags |= __EXEC_ENGINE_PINNED; in eb_pin_engine()
2603 static void eb_unpin_engine(struct i915_execbuffer *eb) in eb_unpin_engine() argument
2605 struct intel_context *ce = eb->context, *child; in eb_unpin_engine()
2607 if (!(eb->args->flags & __EXEC_ENGINE_PINNED)) in eb_unpin_engine()
2610 eb->args->flags &= ~__EXEC_ENGINE_PINNED; in eb_unpin_engine()
2628 eb_select_legacy_ring(struct i915_execbuffer *eb) in eb_select_legacy_ring() argument
2630 struct drm_i915_private *i915 = eb->i915; in eb_select_legacy_ring()
2631 struct drm_i915_gem_execbuffer2 *args = eb->args; in eb_select_legacy_ring()
2639 return -1; in eb_select_legacy_ring()
2643 i915->engine_uabi_class_count[I915_ENGINE_CLASS_VIDEO] > 1) { in eb_select_legacy_ring()
2647 bsd_idx = gen8_dispatch_bsd_engine(i915, eb->file); in eb_select_legacy_ring()
2656 return -1; in eb_select_legacy_ring()
2665 return -1; in eb_select_legacy_ring()
2672 eb_select_engine(struct i915_execbuffer *eb) in eb_select_engine() argument
2679 if (i915_gem_context_user_engines(eb->gem_context)) in eb_select_engine()
2680 idx = eb->args->flags & I915_EXEC_RING_MASK; in eb_select_engine()
2682 idx = eb_select_legacy_ring(eb); in eb_select_engine()
2684 ce = i915_gem_context_get_engine(eb->gem_context, idx); in eb_select_engine()
2689 if (eb->buffer_count < ce->parallel.number_children + 1) { in eb_select_engine()
2693 if (eb->batch_start_offset || eb->args->batch_len) { in eb_select_engine()
2698 eb->num_batches = ce->parallel.number_children + 1; in eb_select_engine()
2703 eb->wakeref = intel_gt_pm_get(ce->engine->gt); in eb_select_engine()
2709 eb->wakeref_gt0 = intel_gt_pm_get(to_gt(gt->i915)); in eb_select_engine()
2737 eb->context = ce; in eb_select_engine()
2738 eb->gt = ce->engine->gt; in eb_select_engine()
2749 intel_gt_pm_put(to_gt(gt->i915), eb->wakeref_gt0); in eb_select_engine()
2751 intel_gt_pm_put(ce->engine->gt, eb->wakeref); in eb_select_engine()
2759 eb_put_engine(struct i915_execbuffer *eb) in eb_put_engine() argument
2763 i915_vm_put(eb->context->vm); in eb_put_engine()
2768 if (eb->gt->info.id) in eb_put_engine()
2769 intel_gt_pm_put(to_gt(eb->gt->i915), eb->wakeref_gt0); in eb_put_engine()
2770 intel_gt_pm_put(eb->context->engine->gt, eb->wakeref); in eb_put_engine()
2771 for_each_child(eb->context, child) in eb_put_engine()
2773 intel_context_put(eb->context); in eb_put_engine()
2788 add_timeline_fence_array(struct i915_execbuffer *eb, in add_timeline_fence_array() argument
2805 SIZE_MAX / sizeof(*f)) - eb->num_fences) in add_timeline_fence_array()
2816 f = krealloc(eb->fences, in add_timeline_fence_array()
2817 (eb->num_fences + nfences) * sizeof(*f), in add_timeline_fence_array()
2822 eb->fences = f; in add_timeline_fence_array()
2823 f += eb->num_fences; in add_timeline_fence_array()
2825 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) & in add_timeline_fence_array()
2845 syncobj = drm_syncobj_find(eb->file, user_fence.handle); in add_timeline_fence_array()
2847 drm_dbg(&eb->i915->drm, in add_timeline_fence_array()
2856 drm_dbg(&eb->i915->drm, in add_timeline_fence_array()
2866 drm_dbg(&eb->i915->drm, in add_timeline_fence_array()
2894 drm_dbg(&eb->i915->drm, in add_timeline_fence_array()
2915 eb->num_fences++; in add_timeline_fence_array()
2921 static int add_fence_array(struct i915_execbuffer *eb) in add_fence_array() argument
2923 struct drm_i915_gem_execbuffer2 *args = eb->args; in add_fence_array()
2938 SIZE_MAX / sizeof(*f) - eb->num_fences)) in add_fence_array()
2945 f = krealloc(eb->fences, in add_fence_array()
2946 (eb->num_fences + num_fences) * sizeof(*f), in add_fence_array()
2951 eb->fences = f; in add_fence_array()
2952 f += eb->num_fences; in add_fence_array()
2964 syncobj = drm_syncobj_find(eb->file, user_fence.handle); in add_fence_array()
2966 drm_dbg(&eb->i915->drm, in add_fence_array()
2974 drm_dbg(&eb->i915->drm, in add_fence_array()
2981 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) & in add_fence_array()
2989 eb->num_fences++; in add_fence_array()
3002 await_fence_array(struct i915_execbuffer *eb, in await_fence_array() argument
3008 for (n = 0; n < eb->num_fences; n++) { in await_fence_array()
3009 if (!eb->fences[n].dma_fence) in await_fence_array()
3012 err = i915_request_await_dma_fence(rq, eb->fences[n].dma_fence); in await_fence_array()
3020 static void signal_fence_array(const struct i915_execbuffer *eb, in signal_fence_array() argument
3025 for (n = 0; n < eb->num_fences; n++) { in signal_fence_array()
3029 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); in signal_fence_array()
3033 if (eb->fences[n].chain_fence) { in signal_fence_array()
3035 eb->fences[n].chain_fence, in signal_fence_array()
3037 eb->fences[n].value); in signal_fence_array()
3042 eb->fences[n].chain_fence = NULL; in signal_fence_array()
3052 struct i915_execbuffer *eb = data; in parse_timeline_fences() local
3058 return add_timeline_fence_array(eb, &timeline_fences); in parse_timeline_fences()
3070 static int eb_request_add(struct i915_execbuffer *eb, struct i915_request *rq, in eb_request_add() argument
3085 if (likely(!intel_context_is_closed(eb->context))) { in eb_request_add()
3086 attr = eb->gem_context->sched; in eb_request_add()
3094 if (intel_context_is_parallel(eb->context)) { in eb_request_add()
3116 static int eb_requests_add(struct i915_execbuffer *eb, int err) in eb_requests_add() argument
3124 for_each_batch_add_order(eb, i) { in eb_requests_add()
3125 struct i915_request *rq = eb->requests[i]; in eb_requests_add()
3129 err |= eb_request_add(eb, rq, err, i == 0); in eb_requests_add()
3141 struct i915_execbuffer *eb) in parse_execbuf2_extensions() argument
3149 if (eb->args->flags & I915_EXEC_FENCE_ARRAY) in parse_execbuf2_extensions()
3158 eb); in parse_execbuf2_extensions()
3161 static void eb_requests_get(struct i915_execbuffer *eb) in eb_requests_get() argument
3165 for_each_batch_create_order(eb, i) { in eb_requests_get()
3166 if (!eb->requests[i]) in eb_requests_get()
3169 i915_request_get(eb->requests[i]); in eb_requests_get()
3173 static void eb_requests_put(struct i915_execbuffer *eb) in eb_requests_put() argument
3177 for_each_batch_create_order(eb, i) { in eb_requests_put()
3178 if (!eb->requests[i]) in eb_requests_put()
3181 i915_request_put(eb->requests[i]); in eb_requests_put()
3186 eb_composite_fence_create(struct i915_execbuffer *eb, int out_fence_fd) in eb_composite_fence_create() argument
3193 GEM_BUG_ON(!intel_context_is_parent(eb->context)); in eb_composite_fence_create()
3195 fences = kmalloc_objs(*fences, eb->num_batches); in eb_composite_fence_create()
3199 for_each_batch_create_order(eb, i) { in eb_composite_fence_create()
3200 fences[i] = &eb->requests[i]->fence; in eb_composite_fence_create()
3202 &eb->requests[i]->fence.flags); in eb_composite_fence_create()
3205 fence_array = dma_fence_array_create(eb->num_batches, in eb_composite_fence_create()
3207 eb->context->parallel.fence_context, in eb_composite_fence_create()
3208 eb->context->parallel.seqno++, in eb_composite_fence_create()
3216 for_each_batch_create_order(eb, i) in eb_composite_fence_create()
3219 if (out_fence_fd != -1) { in eb_composite_fence_create()
3227 eb->composite_fence = &fence_array->base; in eb_composite_fence_create()
3233 eb_fences_add(struct i915_execbuffer *eb, struct i915_request *rq, in eb_fences_add() argument
3239 if (unlikely(eb->gem_context->syncobj)) { in eb_fences_add()
3242 fence = drm_syncobj_fence_get(eb->gem_context->syncobj); in eb_fences_add()
3250 if (eb->args->flags & I915_EXEC_FENCE_SUBMIT) in eb_fences_add()
3258 if (eb->fences) { in eb_fences_add()
3259 err = await_fence_array(eb, rq); in eb_fences_add()
3264 if (intel_context_is_parallel(eb->context)) { in eb_fences_add()
3265 out_fence = eb_composite_fence_create(eb, out_fence_fd); in eb_fences_add()
3268 } else if (out_fence_fd != -1) { in eb_fences_add()
3278 eb_find_context(struct i915_execbuffer *eb, unsigned int context_number) in eb_find_context() argument
3283 return eb->context; in eb_find_context()
3285 for_each_child(eb->context, child) in eb_find_context()
3295 eb_requests_create(struct i915_execbuffer *eb, struct dma_fence *in_fence, in eb_requests_create() argument
3301 for_each_batch_create_order(eb, i) { in eb_requests_create()
3303 eb->requests[i] = i915_request_create(eb_find_context(eb, i)); in eb_requests_create()
3304 if (IS_ERR(eb->requests[i])) { in eb_requests_create()
3305 out_fence = ERR_CAST(eb->requests[i]); in eb_requests_create()
3306 eb->requests[i] = NULL; in eb_requests_create()
3315 if (i + 1 == eb->num_batches) { in eb_requests_create()
3316 out_fence = eb_fences_add(eb, eb->requests[i], in eb_requests_create()
3327 if (eb->batches[i]->vma) in eb_requests_create()
3328 eb->requests[i]->batch_res = in eb_requests_create()
3329 i915_vma_resource_get(eb->batches[i]->vma->resource); in eb_requests_create()
3330 if (eb->batch_pool) { in eb_requests_create()
3331 GEM_BUG_ON(intel_context_is_parallel(eb->context)); in eb_requests_create()
3332 intel_gt_buffer_pool_mark_active(eb->batch_pool, in eb_requests_create()
3333 eb->requests[i]); in eb_requests_create()
3347 struct i915_execbuffer eb; in i915_gem_do_execbuffer() local
3350 int out_fence_fd = -1; in i915_gem_do_execbuffer()
3357 eb.i915 = i915; in i915_gem_do_execbuffer()
3358 eb.file = file; in i915_gem_do_execbuffer()
3359 eb.args = args; in i915_gem_do_execbuffer()
3363 eb.exec = exec; in i915_gem_do_execbuffer()
3364 eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1); in i915_gem_do_execbuffer()
3365 memset(eb.vma, 0, (args->buffer_count + 1) * sizeof(struct eb_vma)); in i915_gem_do_execbuffer()
3367 eb.batch_pool = NULL; in i915_gem_do_execbuffer()
3369 eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; in i915_gem_do_execbuffer()
3370 reloc_cache_init(&eb.reloc_cache, eb.i915); in i915_gem_do_execbuffer()
3372 eb.buffer_count = args->buffer_count; in i915_gem_do_execbuffer()
3373 eb.batch_start_offset = args->batch_start_offset; in i915_gem_do_execbuffer()
3374 eb.trampoline = NULL; in i915_gem_do_execbuffer()
3376 eb.fences = NULL; in i915_gem_do_execbuffer()
3377 eb.num_fences = 0; in i915_gem_do_execbuffer()
3379 eb_capture_list_clear(&eb); in i915_gem_do_execbuffer()
3381 memset(eb.requests, 0, sizeof(struct i915_request *) * in i915_gem_do_execbuffer()
3382 ARRAY_SIZE(eb.requests)); in i915_gem_do_execbuffer()
3383 eb.composite_fence = NULL; in i915_gem_do_execbuffer()
3385 eb.batch_flags = 0; in i915_gem_do_execbuffer()
3397 eb.batch_flags |= I915_DISPATCH_SECURE; in i915_gem_do_execbuffer()
3400 eb.batch_flags |= I915_DISPATCH_PINNED; in i915_gem_do_execbuffer()
3402 err = parse_execbuf2_extensions(args, &eb); in i915_gem_do_execbuffer()
3406 err = add_fence_array(&eb); in i915_gem_do_execbuffer()
3431 err = eb_create(&eb); in i915_gem_do_execbuffer()
3435 GEM_BUG_ON(!eb.lut_size); in i915_gem_do_execbuffer()
3437 err = eb_select_context(&eb); in i915_gem_do_execbuffer()
3441 err = eb_select_engine(&eb); in i915_gem_do_execbuffer()
3445 err = eb_lookup_vmas(&eb); in i915_gem_do_execbuffer()
3447 eb_release_vmas(&eb, true); in i915_gem_do_execbuffer()
3451 i915_gem_ww_ctx_init(&eb.ww, true); in i915_gem_do_execbuffer()
3453 err = eb_relocate_parse(&eb); in i915_gem_do_execbuffer()
3466 ww_acquire_done(&eb.ww.ctx); in i915_gem_do_execbuffer()
3467 err = eb_capture_stage(&eb); in i915_gem_do_execbuffer()
3471 out_fence = eb_requests_create(&eb, in_fence, out_fence_fd); in i915_gem_do_execbuffer()
3475 if (eb.requests[0]) in i915_gem_do_execbuffer()
3481 err = eb_submit(&eb); in i915_gem_do_execbuffer()
3484 eb_requests_get(&eb); in i915_gem_do_execbuffer()
3485 err = eb_requests_add(&eb, err); in i915_gem_do_execbuffer()
3487 if (eb.fences) in i915_gem_do_execbuffer()
3488 signal_fence_array(&eb, eb.composite_fence ? in i915_gem_do_execbuffer()
3489 eb.composite_fence : in i915_gem_do_execbuffer()
3490 &eb.requests[0]->fence); in i915_gem_do_execbuffer()
3492 if (unlikely(eb.gem_context->syncobj)) { in i915_gem_do_execbuffer()
3493 drm_syncobj_replace_fence(eb.gem_context->syncobj, in i915_gem_do_execbuffer()
3494 eb.composite_fence ? in i915_gem_do_execbuffer()
3495 eb.composite_fence : in i915_gem_do_execbuffer()
3496 &eb.requests[0]->fence); in i915_gem_do_execbuffer()
3504 out_fence_fd = -1; in i915_gem_do_execbuffer()
3510 if (!out_fence && eb.composite_fence) in i915_gem_do_execbuffer()
3511 dma_fence_put(eb.composite_fence); in i915_gem_do_execbuffer()
3513 eb_requests_put(&eb); in i915_gem_do_execbuffer()
3516 eb_release_vmas(&eb, true); in i915_gem_do_execbuffer()
3518 i915_gem_ww_ctx_fini(&eb.ww); in i915_gem_do_execbuffer()
3520 if (eb.batch_pool) in i915_gem_do_execbuffer()
3521 intel_gt_buffer_pool_put(eb.batch_pool); in i915_gem_do_execbuffer()
3523 eb_put_engine(&eb); in i915_gem_do_execbuffer()
3525 i915_gem_context_put(eb.gem_context); in i915_gem_do_execbuffer()
3527 eb_destroy(&eb); in i915_gem_do_execbuffer()
3529 if (out_fence_fd != -1) in i915_gem_do_execbuffer()
3534 put_fence_array(eb.fences, eb.num_fences); in i915_gem_do_execbuffer()
3553 return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1); in check_buffer_count()
3579 * exec2_list array, and the eventual eb.vma array introduced in in i915_gem_execbuffer2_ioctl()
3584 * However, the eb.vma part is explicitly zeroed later in in i915_gem_execbuffer2_ioctl()