Lines Matching full:stream
137 static u32 xe_oa_circ_diff(struct xe_oa_stream *stream, u32 tail, u32 head) in xe_oa_circ_diff() argument
140 tail + stream->oa_buffer.circ_size - head; in xe_oa_circ_diff()
143 static u32 xe_oa_circ_incr(struct xe_oa_stream *stream, u32 ptr, u32 n) in xe_oa_circ_incr() argument
145 return ptr + n >= stream->oa_buffer.circ_size ? in xe_oa_circ_incr()
146 ptr + n - stream->oa_buffer.circ_size : ptr + n; in xe_oa_circ_incr()
192 static const struct xe_oa_regs *__oa_regs(struct xe_oa_stream *stream) in __oa_regs() argument
194 return &stream->hwe->oa_unit->regs; in __oa_regs()
197 static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream) in xe_oa_hw_tail_read() argument
199 return xe_mmio_read32(&stream->gt->mmio, __oa_regs(stream)->oa_tail_ptr) & in xe_oa_hw_tail_read()
206 static u64 oa_report_id(struct xe_oa_stream *stream, void *report) in oa_report_id() argument
208 return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report; in oa_report_id()
211 static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report) in oa_report_id_clear() argument
213 if (oa_report_header_64bit(stream)) in oa_report_id_clear()
219 static u64 oa_timestamp(struct xe_oa_stream *stream, void *report) in oa_timestamp() argument
221 return oa_report_header_64bit(stream) ? in oa_timestamp()
226 static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report) in oa_timestamp_clear() argument
228 if (oa_report_header_64bit(stream)) in oa_timestamp_clear()
234 static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream) in xe_oa_buffer_check_unlocked() argument
236 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); in xe_oa_buffer_check_unlocked()
238 int report_size = stream->oa_buffer.format->size; in xe_oa_buffer_check_unlocked()
241 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_buffer_check_unlocked()
243 hw_tail = xe_oa_hw_tail_read(stream); in xe_oa_buffer_check_unlocked()
251 partial_report_size = xe_oa_circ_diff(stream, hw_tail, stream->oa_buffer.tail); in xe_oa_buffer_check_unlocked()
255 hw_tail = xe_oa_circ_diff(stream, hw_tail, partial_report_size); in xe_oa_buffer_check_unlocked()
260 * Walk the stream backward until we find a report with report id and timestamp in xe_oa_buffer_check_unlocked()
267 while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) { in xe_oa_buffer_check_unlocked()
268 void *report = stream->oa_buffer.vaddr + tail; in xe_oa_buffer_check_unlocked()
270 if (oa_report_id(stream, report) || oa_timestamp(stream, report)) in xe_oa_buffer_check_unlocked()
273 tail = xe_oa_circ_diff(stream, tail, report_size); in xe_oa_buffer_check_unlocked()
276 if (xe_oa_circ_diff(stream, hw_tail, tail) > report_size) in xe_oa_buffer_check_unlocked()
277 drm_dbg(&stream->oa->xe->drm, in xe_oa_buffer_check_unlocked()
279 stream->oa_buffer.head, tail, hw_tail); in xe_oa_buffer_check_unlocked()
281 stream->oa_buffer.tail = tail; in xe_oa_buffer_check_unlocked()
283 available = xe_oa_circ_diff(stream, stream->oa_buffer.tail, stream->oa_buffer.head); in xe_oa_buffer_check_unlocked()
284 stream->pollin = available >= stream->wait_num_reports * report_size; in xe_oa_buffer_check_unlocked()
286 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_buffer_check_unlocked()
288 return stream->pollin; in xe_oa_buffer_check_unlocked()
293 struct xe_oa_stream *stream = in xe_oa_poll_check_timer_cb() local
294 container_of(hrtimer, typeof(*stream), poll_check_timer); in xe_oa_poll_check_timer_cb()
296 if (xe_oa_buffer_check_unlocked(stream)) in xe_oa_poll_check_timer_cb()
297 wake_up(&stream->poll_wq); in xe_oa_poll_check_timer_cb()
299 hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period_ns)); in xe_oa_poll_check_timer_cb()
304 static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf, in xe_oa_append_report() argument
307 int report_size = stream->oa_buffer.format->size; in xe_oa_append_report()
316 oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; in xe_oa_append_report()
324 if (copy_to_user(buf, stream->oa_buffer.vaddr, in xe_oa_append_report()
336 static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf, in xe_oa_append_reports() argument
339 int report_size = stream->oa_buffer.format->size; in xe_oa_append_reports()
340 u8 *oa_buf_base = stream->oa_buffer.vaddr; in xe_oa_append_reports()
341 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); in xe_oa_append_reports()
347 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
348 head = stream->oa_buffer.head; in xe_oa_append_reports()
349 tail = stream->oa_buffer.tail; in xe_oa_append_reports()
350 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
352 xe_assert(stream->oa->xe, in xe_oa_append_reports()
353 head < stream->oa_buffer.circ_size && tail < stream->oa_buffer.circ_size); in xe_oa_append_reports()
355 for (; xe_oa_circ_diff(stream, tail, head); in xe_oa_append_reports()
356 head = xe_oa_circ_incr(stream, head, report_size)) { in xe_oa_append_reports()
359 ret = xe_oa_append_report(stream, buf, count, offset, report); in xe_oa_append_reports()
363 if (!(stream->oa_buffer.circ_size % report_size)) { in xe_oa_append_reports()
365 oa_report_id_clear(stream, (void *)report); in xe_oa_append_reports()
366 oa_timestamp_clear(stream, (void *)report); in xe_oa_append_reports()
368 u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; in xe_oa_append_reports()
382 struct xe_reg oaheadptr = __oa_regs(stream)->oa_head_ptr; in xe_oa_append_reports()
384 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
385 xe_mmio_write32(&stream->gt->mmio, oaheadptr, in xe_oa_append_reports()
387 stream->oa_buffer.head = head; in xe_oa_append_reports()
388 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
394 static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream) in xe_oa_init_oa_buffer() argument
396 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); in xe_oa_init_oa_buffer()
397 int size_exponent = __ffs(stream->oa_buffer.bo->size); in xe_oa_init_oa_buffer()
399 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_init_oa_buffer()
409 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_init_oa_buffer()
411 xe_mmio_write32(mmio, __oa_regs(stream)->oa_status, 0); in xe_oa_init_oa_buffer()
412 xe_mmio_write32(mmio, __oa_regs(stream)->oa_head_ptr, in xe_oa_init_oa_buffer()
414 stream->oa_buffer.head = 0; in xe_oa_init_oa_buffer()
419 xe_mmio_write32(mmio, __oa_regs(stream)->oa_buffer, oa_buf); in xe_oa_init_oa_buffer()
420 xe_mmio_write32(mmio, __oa_regs(stream)->oa_tail_ptr, in xe_oa_init_oa_buffer()
424 stream->oa_buffer.tail = 0; in xe_oa_init_oa_buffer()
426 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_init_oa_buffer()
429 memset(stream->oa_buffer.vaddr, 0, stream->oa_buffer.bo->size); in xe_oa_init_oa_buffer()
439 static u32 __oa_ccs_select(struct xe_oa_stream *stream) in __oa_ccs_select() argument
443 if (stream->hwe->class != XE_ENGINE_CLASS_COMPUTE) in __oa_ccs_select()
446 val = REG_FIELD_PREP(OAG_OACONTROL_OA_CCS_SELECT_MASK, stream->hwe->instance); in __oa_ccs_select()
447 xe_assert(stream->oa->xe, in __oa_ccs_select()
448 REG_FIELD_GET(OAG_OACONTROL_OA_CCS_SELECT_MASK, val) == stream->hwe->instance); in __oa_ccs_select()
452 static u32 __oactrl_used_bits(struct xe_oa_stream *stream) in __oactrl_used_bits() argument
454 return stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG ? in __oactrl_used_bits()
458 static void xe_oa_enable(struct xe_oa_stream *stream) in xe_oa_enable() argument
460 const struct xe_oa_format *format = stream->oa_buffer.format; in xe_oa_enable()
465 * BSpec: 46822: Bit 0. Even if stream->sample is 0, for OAR to function, the OA in xe_oa_enable()
468 xe_oa_init_oa_buffer(stream); in xe_oa_enable()
470 regs = __oa_regs(stream); in xe_oa_enable()
472 __oa_ccs_select(stream) | OAG_OACONTROL_OA_COUNTER_ENABLE; in xe_oa_enable()
474 if (GRAPHICS_VER(stream->oa->xe) >= 20 && in xe_oa_enable()
475 stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG) in xe_oa_enable()
478 xe_mmio_rmw32(&stream->gt->mmio, regs->oa_ctrl, __oactrl_used_bits(stream), val); in xe_oa_enable()
481 static void xe_oa_disable(struct xe_oa_stream *stream) in xe_oa_disable() argument
483 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_disable()
485 xe_mmio_rmw32(mmio, __oa_regs(stream)->oa_ctrl, __oactrl_used_bits(stream), 0); in xe_oa_disable()
486 if (xe_mmio_wait32(mmio, __oa_regs(stream)->oa_ctrl, in xe_oa_disable()
488 drm_err(&stream->oa->xe->drm, in xe_oa_disable()
491 if (GRAPHICS_VERx100(stream->oa->xe) <= 1270 && GRAPHICS_VERx100(stream->oa->xe) != 1260) { in xe_oa_disable()
495 drm_err(&stream->oa->xe->drm, in xe_oa_disable()
500 static int xe_oa_wait_unlocked(struct xe_oa_stream *stream) in xe_oa_wait_unlocked() argument
503 if (!stream->periodic) in xe_oa_wait_unlocked()
506 return wait_event_interruptible(stream->poll_wq, in xe_oa_wait_unlocked()
507 xe_oa_buffer_check_unlocked(stream)); in xe_oa_wait_unlocked()
513 static int __xe_oa_read(struct xe_oa_stream *stream, char __user *buf, in __xe_oa_read() argument
517 stream->oa_status = xe_mmio_rmw32(&stream->gt->mmio, __oa_regs(stream)->oa_status, in __xe_oa_read()
521 * @DRM_XE_OBSERVATION_IOCTL_STATUS observation stream fd ioctl in __xe_oa_read()
523 if (stream->oa_status & OASTATUS_RELEVANT_BITS) in __xe_oa_read()
526 return xe_oa_append_reports(stream, buf, count, offset); in __xe_oa_read()
532 struct xe_oa_stream *stream = file->private_data; in xe_oa_read() local
537 if (!stream->enabled || !stream->sample) in xe_oa_read()
542 ret = xe_oa_wait_unlocked(stream); in xe_oa_read()
546 mutex_lock(&stream->stream_lock); in xe_oa_read()
547 ret = __xe_oa_read(stream, buf, count, &offset); in xe_oa_read()
548 mutex_unlock(&stream->stream_lock); in xe_oa_read()
551 mutex_lock(&stream->stream_lock); in xe_oa_read()
552 ret = __xe_oa_read(stream, buf, count, &offset); in xe_oa_read()
553 mutex_unlock(&stream->stream_lock); in xe_oa_read()
566 stream->pollin = false; in xe_oa_read()
572 static __poll_t xe_oa_poll_locked(struct xe_oa_stream *stream, in xe_oa_poll_locked() argument
577 poll_wait(file, &stream->poll_wq, wait); in xe_oa_poll_locked()
585 if (stream->pollin) in xe_oa_poll_locked()
593 struct xe_oa_stream *stream = file->private_data; in xe_oa_poll() local
596 mutex_lock(&stream->stream_lock); in xe_oa_poll()
597 ret = xe_oa_poll_locked(stream, file, wait); in xe_oa_poll()
598 mutex_unlock(&stream->stream_lock); in xe_oa_poll()
619 static struct dma_fence *xe_oa_submit_bb(struct xe_oa_stream *stream, enum xe_oa_submit_deps deps, in xe_oa_submit_bb() argument
622 struct xe_exec_queue *q = stream->exec_q ?: stream->k_exec_q; in xe_oa_submit_bb()
637 for (int i = 0; i < stream->num_syncs && !err; i++) in xe_oa_submit_bb()
638 err = xe_sync_entry_add_deps(&stream->syncs[i], job); in xe_oa_submit_bb()
640 drm_dbg(&stream->oa->xe->drm, "xe_sync_entry_add_deps err %d\n", err); in xe_oa_submit_bb()
689 static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream) in xe_oa_free_oa_buffer() argument
691 xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); in xe_oa_free_oa_buffer()
694 static void xe_oa_free_configs(struct xe_oa_stream *stream) in xe_oa_free_configs() argument
698 xe_oa_config_put(stream->oa_config); in xe_oa_free_configs()
699 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) in xe_oa_free_configs()
700 free_oa_config_bo(oa_bo, stream->last_fence); in xe_oa_free_configs()
701 dma_fence_put(stream->last_fence); in xe_oa_free_configs()
704 static int xe_oa_load_with_lri(struct xe_oa_stream *stream, struct xe_oa_reg *reg_lri, u32 count) in xe_oa_load_with_lri() argument
710 bb = xe_bb_new(stream->gt, 2 * count + 1, false); in xe_oa_load_with_lri()
718 fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_NO_DEPS, bb); in xe_oa_load_with_lri()
733 static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable) in xe_oa_configure_oar_context() argument
735 const struct xe_oa_format *format = stream->oa_buffer.format; in xe_oa_configure_oar_context()
741 OACTXCONTROL(stream->hwe->mmio_base), in xe_oa_configure_oar_context()
749 RING_CONTEXT_CONTROL(stream->hwe->mmio_base), in xe_oa_configure_oar_context()
755 return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri)); in xe_oa_configure_oar_context()
758 static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable) in xe_oa_configure_oac_context() argument
760 const struct xe_oa_format *format = stream->oa_buffer.format; in xe_oa_configure_oac_context()
765 OACTXCONTROL(stream->hwe->mmio_base), in xe_oa_configure_oac_context()
773 RING_CONTEXT_CONTROL(stream->hwe->mmio_base), in xe_oa_configure_oac_context()
781 xe_mmio_write32(&stream->gt->mmio, __oa_regs(stream)->oa_ctrl, in xe_oa_configure_oac_context()
782 __oa_ccs_select(stream)); in xe_oa_configure_oac_context()
784 return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri)); in xe_oa_configure_oac_context()
787 static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable) in xe_oa_configure_oa_context() argument
789 switch (stream->hwe->class) { in xe_oa_configure_oa_context()
791 return xe_oa_configure_oar_context(stream, enable); in xe_oa_configure_oa_context()
793 return xe_oa_configure_oac_context(stream, enable); in xe_oa_configure_oa_context()
802 static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable) in oag_configure_mmio_trigger() argument
805 enable && stream && stream->sample ? in oag_configure_mmio_trigger()
809 static void xe_oa_disable_metric_set(struct xe_oa_stream *stream) in xe_oa_disable_metric_set() argument
811 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_disable_metric_set()
818 if (stream->oa->xe->info.platform == XE_DG2) { in xe_oa_disable_metric_set()
819 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, in xe_oa_disable_metric_set()
821 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, in xe_oa_disable_metric_set()
825 xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug, in xe_oa_disable_metric_set()
826 oag_configure_mmio_trigger(stream, false)); in xe_oa_disable_metric_set()
829 if (stream->exec_q) in xe_oa_disable_metric_set()
830 xe_oa_configure_oa_context(stream, false); in xe_oa_disable_metric_set()
836 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); in xe_oa_disable_metric_set()
842 static void xe_oa_stream_destroy(struct xe_oa_stream *stream) in xe_oa_stream_destroy() argument
844 struct xe_oa_unit *u = stream->hwe->oa_unit; in xe_oa_stream_destroy()
845 struct xe_gt *gt = stream->hwe->gt; in xe_oa_stream_destroy()
847 if (WARN_ON(stream != u->exclusive_stream)) in xe_oa_stream_destroy()
852 mutex_destroy(&stream->stream_lock); in xe_oa_stream_destroy()
854 xe_oa_disable_metric_set(stream); in xe_oa_stream_destroy()
855 xe_exec_queue_put(stream->k_exec_q); in xe_oa_stream_destroy()
857 xe_oa_free_oa_buffer(stream); in xe_oa_stream_destroy()
860 xe_pm_runtime_put(stream->oa->xe); in xe_oa_stream_destroy()
863 if (stream->override_gucrc) in xe_oa_stream_destroy()
866 xe_oa_free_configs(stream); in xe_oa_stream_destroy()
867 xe_file_put(stream->xef); in xe_oa_stream_destroy()
870 static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream, size_t size) in xe_oa_alloc_oa_buffer() argument
874 bo = xe_bo_create_pin_map(stream->oa->xe, stream->gt->tile, NULL, in xe_oa_alloc_oa_buffer()
880 stream->oa_buffer.bo = bo; in xe_oa_alloc_oa_buffer()
882 xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0); in xe_oa_alloc_oa_buffer()
883 stream->oa_buffer.vaddr = bo->vmap.vaddr; in xe_oa_alloc_oa_buffer()
888 __xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config) in __xe_oa_alloc_config_buffer() argument
901 bb = xe_bb_new(stream->gt, config_length, false); in __xe_oa_alloc_config_buffer()
909 llist_add(&oa_bo->node, &stream->oa_config_bos); in __xe_oa_alloc_config_buffer()
918 xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config) in xe_oa_alloc_config_buffer() argument
922 /* Look for the buffer in the already allocated BOs attached to the stream */ in xe_oa_alloc_config_buffer()
923 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { in xe_oa_alloc_config_buffer()
930 oa_bo = __xe_oa_alloc_config_buffer(stream, oa_config); in xe_oa_alloc_config_buffer()
935 static void xe_oa_update_last_fence(struct xe_oa_stream *stream, struct dma_fence *fence) in xe_oa_update_last_fence() argument
937 dma_fence_put(stream->last_fence); in xe_oa_update_last_fence()
938 stream->last_fence = dma_fence_get(fence); in xe_oa_update_last_fence()
978 static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config *config) in xe_oa_emit_oa_config() argument
992 oa_bo = xe_oa_alloc_config_buffer(stream, config); in xe_oa_emit_oa_config()
999 fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_ADD_DEPS, oa_bo->bb); in xe_oa_emit_oa_config()
1009 for (i = 0; i < stream->num_syncs; i++) { in xe_oa_emit_oa_config()
1010 if (stream->syncs[i].flags & DRM_XE_SYNC_FLAG_SIGNAL) in xe_oa_emit_oa_config()
1012 xe_sync_entry_signal(&stream->syncs[i], &ofence->base); in xe_oa_emit_oa_config()
1020 xe_oa_update_last_fence(stream, fence); in xe_oa_emit_oa_config()
1024 xe_gt_assert(stream->gt, !err || err == -ENOENT); in xe_oa_emit_oa_config()
1035 for (i = 0; i < stream->num_syncs; i++) in xe_oa_emit_oa_config()
1036 xe_sync_entry_cleanup(&stream->syncs[i]); in xe_oa_emit_oa_config()
1037 kfree(stream->syncs); in xe_oa_emit_oa_config()
1045 static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream) in oag_report_ctx_switches() argument
1049 stream->sample ? in oag_report_ctx_switches()
1053 static u32 oag_buf_size_select(const struct xe_oa_stream *stream) in oag_buf_size_select() argument
1056 stream->oa_buffer.bo->size > SZ_16M ? in oag_buf_size_select()
1060 static int xe_oa_enable_metric_set(struct xe_oa_stream *stream) in xe_oa_enable_metric_set() argument
1062 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_enable_metric_set()
1071 if (stream->oa->xe->info.platform == XE_DG2) { in xe_oa_enable_metric_set()
1072 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, in xe_oa_enable_metric_set()
1074 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, in xe_oa_enable_metric_set()
1082 if (GRAPHICS_VER(stream->oa->xe) >= 20) in xe_oa_enable_metric_set()
1089 xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug, in xe_oa_enable_metric_set()
1091 oag_report_ctx_switches(stream) | in xe_oa_enable_metric_set()
1092 oag_buf_size_select(stream) | in xe_oa_enable_metric_set()
1093 oag_configure_mmio_trigger(stream, true)); in xe_oa_enable_metric_set()
1095 xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ? in xe_oa_enable_metric_set()
1099 stream->period_exponent)) : 0); in xe_oa_enable_metric_set()
1107 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); in xe_oa_enable_metric_set()
1112 if (stream->exec_q) { in xe_oa_enable_metric_set()
1113 ret = xe_oa_configure_oa_context(stream, true); in xe_oa_enable_metric_set()
1118 return xe_oa_emit_oa_config(stream, stream->oa_config); in xe_oa_enable_metric_set()
1403 static void xe_oa_stream_enable(struct xe_oa_stream *stream) in xe_oa_stream_enable() argument
1405 stream->pollin = false; in xe_oa_stream_enable()
1407 xe_oa_enable(stream); in xe_oa_stream_enable()
1409 if (stream->sample) in xe_oa_stream_enable()
1410 hrtimer_start(&stream->poll_check_timer, in xe_oa_stream_enable()
1411 ns_to_ktime(stream->poll_period_ns), in xe_oa_stream_enable()
1415 static void xe_oa_stream_disable(struct xe_oa_stream *stream) in xe_oa_stream_disable() argument
1417 xe_oa_disable(stream); in xe_oa_stream_disable()
1419 if (stream->sample) in xe_oa_stream_disable()
1420 hrtimer_cancel(&stream->poll_check_timer); in xe_oa_stream_disable()
1423 static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream) in xe_oa_enable_preempt_timeslice() argument
1425 struct xe_exec_queue *q = stream->exec_q; in xe_oa_enable_preempt_timeslice()
1429 ret1 = q->ops->set_timeslice(q, stream->hwe->eclass->sched_props.timeslice_us); in xe_oa_enable_preempt_timeslice()
1430 ret2 = q->ops->set_preempt_timeout(q, stream->hwe->eclass->sched_props.preempt_timeout_us); in xe_oa_enable_preempt_timeslice()
1435 drm_dbg(&stream->oa->xe->drm, "%s failed ret1 %d ret2 %d\n", __func__, ret1, ret2); in xe_oa_enable_preempt_timeslice()
1439 static int xe_oa_disable_preempt_timeslice(struct xe_oa_stream *stream) in xe_oa_disable_preempt_timeslice() argument
1441 struct xe_exec_queue *q = stream->exec_q; in xe_oa_disable_preempt_timeslice()
1455 xe_oa_enable_preempt_timeslice(stream); in xe_oa_disable_preempt_timeslice()
1456 drm_dbg(&stream->oa->xe->drm, "%s failed %d\n", __func__, ret); in xe_oa_disable_preempt_timeslice()
1460 static int xe_oa_enable_locked(struct xe_oa_stream *stream) in xe_oa_enable_locked() argument
1462 if (stream->enabled) in xe_oa_enable_locked()
1465 if (stream->no_preempt) { in xe_oa_enable_locked()
1466 int ret = xe_oa_disable_preempt_timeslice(stream); in xe_oa_enable_locked()
1472 xe_oa_stream_enable(stream); in xe_oa_enable_locked()
1474 stream->enabled = true; in xe_oa_enable_locked()
1478 static int xe_oa_disable_locked(struct xe_oa_stream *stream) in xe_oa_disable_locked() argument
1482 if (!stream->enabled) in xe_oa_disable_locked()
1485 xe_oa_stream_disable(stream); in xe_oa_disable_locked()
1487 if (stream->no_preempt) in xe_oa_disable_locked()
1488 ret = xe_oa_enable_preempt_timeslice(stream); in xe_oa_disable_locked()
1490 stream->enabled = false; in xe_oa_disable_locked()
1494 static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg) in xe_oa_config_locked() argument
1497 long ret = stream->oa_config->id; in xe_oa_config_locked()
1501 err = xe_oa_user_extensions(stream->oa, XE_OA_USER_EXTN_FROM_CONFIG, arg, 0, ¶m); in xe_oa_config_locked()
1505 config = xe_oa_get_oa_config(stream->oa, param.metric_set); in xe_oa_config_locked()
1509 param.xef = stream->xef; in xe_oa_config_locked()
1510 err = xe_oa_parse_syncs(stream->oa, ¶m); in xe_oa_config_locked()
1514 stream->num_syncs = param.num_syncs; in xe_oa_config_locked()
1515 stream->syncs = param.syncs; in xe_oa_config_locked()
1517 err = xe_oa_emit_oa_config(stream, config); in xe_oa_config_locked()
1519 config = xchg(&stream->oa_config, config); in xe_oa_config_locked()
1520 drm_dbg(&stream->oa->xe->drm, "changed to oa config uuid=%s\n", in xe_oa_config_locked()
1521 stream->oa_config->uuid); in xe_oa_config_locked()
1530 static long xe_oa_status_locked(struct xe_oa_stream *stream, unsigned long arg) in xe_oa_status_locked() argument
1536 if (stream->oa_status & OASTATUS_REPORT_LOST) in xe_oa_status_locked()
1538 if (stream->oa_status & OASTATUS_BUFFER_OVERFLOW) in xe_oa_status_locked()
1540 if (stream->oa_status & OASTATUS_COUNTER_OVERFLOW) in xe_oa_status_locked()
1542 if (stream->oa_status & OASTATUS_MMIO_TRG_Q_FULL) in xe_oa_status_locked()
1551 static long xe_oa_info_locked(struct xe_oa_stream *stream, unsigned long arg) in xe_oa_info_locked() argument
1553 struct drm_xe_oa_stream_info info = { .oa_buf_size = stream->oa_buffer.bo->size, }; in xe_oa_info_locked()
1562 static long xe_oa_ioctl_locked(struct xe_oa_stream *stream, in xe_oa_ioctl_locked() argument
1568 return xe_oa_enable_locked(stream); in xe_oa_ioctl_locked()
1570 return xe_oa_disable_locked(stream); in xe_oa_ioctl_locked()
1572 return xe_oa_config_locked(stream, arg); in xe_oa_ioctl_locked()
1574 return xe_oa_status_locked(stream, arg); in xe_oa_ioctl_locked()
1576 return xe_oa_info_locked(stream, arg); in xe_oa_ioctl_locked()
1586 struct xe_oa_stream *stream = file->private_data; in xe_oa_ioctl() local
1589 mutex_lock(&stream->stream_lock); in xe_oa_ioctl()
1590 ret = xe_oa_ioctl_locked(stream, cmd, arg); in xe_oa_ioctl()
1591 mutex_unlock(&stream->stream_lock); in xe_oa_ioctl()
1596 static void xe_oa_destroy_locked(struct xe_oa_stream *stream) in xe_oa_destroy_locked() argument
1598 if (stream->enabled) in xe_oa_destroy_locked()
1599 xe_oa_disable_locked(stream); in xe_oa_destroy_locked()
1601 xe_oa_stream_destroy(stream); in xe_oa_destroy_locked()
1603 if (stream->exec_q) in xe_oa_destroy_locked()
1604 xe_exec_queue_put(stream->exec_q); in xe_oa_destroy_locked()
1606 kfree(stream); in xe_oa_destroy_locked()
1611 struct xe_oa_stream *stream = file->private_data; in xe_oa_release() local
1612 struct xe_gt *gt = stream->gt; in xe_oa_release()
1616 xe_oa_destroy_locked(stream); in xe_oa_release()
1620 /* Release the reference the OA stream kept on the driver */ in xe_oa_release()
1628 struct xe_oa_stream *stream = file->private_data; in xe_oa_mmap() local
1629 struct xe_bo *bo = stream->oa_buffer.bo; in xe_oa_mmap()
1634 drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n"); in xe_oa_mmap()
1639 if (vma->vm_end - vma->vm_start != stream->oa_buffer.bo->size) { in xe_oa_mmap()
1640 drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n"); in xe_oa_mmap()
1649 drm_dbg(&stream->oa->xe->drm, "mmap must be read only\n"); in xe_oa_mmap()
1655 xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == vma_pages(vma)); in xe_oa_mmap()
1677 static int xe_oa_stream_init(struct xe_oa_stream *stream, in xe_oa_stream_init() argument
1685 stream->exec_q = param->exec_q; in xe_oa_stream_init()
1686 stream->poll_period_ns = DEFAULT_POLL_PERIOD_NS; in xe_oa_stream_init()
1687 stream->hwe = param->hwe; in xe_oa_stream_init()
1688 stream->gt = stream->hwe->gt; in xe_oa_stream_init()
1689 stream->oa_buffer.format = &stream->oa->oa_formats[param->oa_format]; in xe_oa_stream_init()
1691 stream->sample = param->sample; in xe_oa_stream_init()
1692 stream->periodic = param->period_exponent >= 0; in xe_oa_stream_init()
1693 stream->period_exponent = param->period_exponent; in xe_oa_stream_init()
1694 stream->no_preempt = param->no_preempt; in xe_oa_stream_init()
1695 stream->wait_num_reports = param->wait_num_reports; in xe_oa_stream_init()
1697 stream->xef = xe_file_get(param->xef); in xe_oa_stream_init()
1698 stream->num_syncs = param->num_syncs; in xe_oa_stream_init()
1699 stream->syncs = param->syncs; in xe_oa_stream_init()
1706 if (GRAPHICS_VER(stream->oa->xe) >= 20 && in xe_oa_stream_init()
1707 stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG && stream->sample) in xe_oa_stream_init()
1708 stream->oa_buffer.circ_size = in xe_oa_stream_init()
1710 param->oa_buffer_size % stream->oa_buffer.format->size; in xe_oa_stream_init()
1712 stream->oa_buffer.circ_size = param->oa_buffer_size; in xe_oa_stream_init()
1714 stream->oa_config = xe_oa_get_oa_config(stream->oa, param->metric_set); in xe_oa_stream_init()
1715 if (!stream->oa_config) { in xe_oa_stream_init()
1716 drm_dbg(&stream->oa->xe->drm, "Invalid OA config id=%i\n", param->metric_set); in xe_oa_stream_init()
1727 if (stream->oa->xe->info.platform == XE_PVC) { in xe_oa_stream_init()
1733 stream->override_gucrc = true; in xe_oa_stream_init()
1737 xe_pm_runtime_get(stream->oa->xe); in xe_oa_stream_init()
1744 ret = xe_oa_alloc_oa_buffer(stream, param->oa_buffer_size); in xe_oa_stream_init()
1748 stream->k_exec_q = xe_exec_queue_create(stream->oa->xe, NULL, in xe_oa_stream_init()
1749 BIT(stream->hwe->logical_instance), 1, in xe_oa_stream_init()
1750 stream->hwe, EXEC_QUEUE_FLAG_KERNEL, 0); in xe_oa_stream_init()
1751 if (IS_ERR(stream->k_exec_q)) { in xe_oa_stream_init()
1752 ret = PTR_ERR(stream->k_exec_q); in xe_oa_stream_init()
1753 drm_err(&stream->oa->xe->drm, "gt%d, hwe %s, xe_exec_queue_create failed=%d", in xe_oa_stream_init()
1754 stream->gt->info.id, stream->hwe->name, ret); in xe_oa_stream_init()
1758 ret = xe_oa_enable_metric_set(stream); in xe_oa_stream_init()
1760 drm_dbg(&stream->oa->xe->drm, "Unable to enable metric set\n"); in xe_oa_stream_init()
1764 drm_dbg(&stream->oa->xe->drm, "opening stream oa config uuid=%s\n", in xe_oa_stream_init()
1765 stream->oa_config->uuid); in xe_oa_stream_init()
1767 WRITE_ONCE(u->exclusive_stream, stream); in xe_oa_stream_init()
1769 hrtimer_init(&stream->poll_check_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); in xe_oa_stream_init()
1770 stream->poll_check_timer.function = xe_oa_poll_check_timer_cb; in xe_oa_stream_init()
1771 init_waitqueue_head(&stream->poll_wq); in xe_oa_stream_init()
1773 spin_lock_init(&stream->oa_buffer.ptr_lock); in xe_oa_stream_init()
1774 mutex_init(&stream->stream_lock); in xe_oa_stream_init()
1779 xe_oa_disable_metric_set(stream); in xe_oa_stream_init()
1780 xe_exec_queue_put(stream->k_exec_q); in xe_oa_stream_init()
1782 xe_oa_free_oa_buffer(stream); in xe_oa_stream_init()
1785 xe_pm_runtime_put(stream->oa->xe); in xe_oa_stream_init()
1786 if (stream->override_gucrc) in xe_oa_stream_init()
1789 xe_oa_free_configs(stream); in xe_oa_stream_init()
1791 xe_file_put(stream->xef); in xe_oa_stream_init()
1798 struct xe_oa_stream *stream; in xe_oa_stream_open_ioctl_locked() local
1809 stream = kzalloc(sizeof(*stream), GFP_KERNEL); in xe_oa_stream_open_ioctl_locked()
1810 if (!stream) { in xe_oa_stream_open_ioctl_locked()
1815 stream->oa = oa; in xe_oa_stream_open_ioctl_locked()
1816 ret = xe_oa_stream_init(stream, param); in xe_oa_stream_open_ioctl_locked()
1821 ret = xe_oa_enable_locked(stream); in xe_oa_stream_open_ioctl_locked()
1826 stream_fd = anon_inode_getfd("[xe_oa]", &xe_oa_fops, stream, 0); in xe_oa_stream_open_ioctl_locked()
1833 drm_dev_get(&stream->oa->xe->drm); in xe_oa_stream_open_ioctl_locked()
1838 xe_oa_disable_locked(stream); in xe_oa_stream_open_ioctl_locked()
1840 xe_oa_stream_destroy(stream); in xe_oa_stream_open_ioctl_locked()
1842 kfree(stream); in xe_oa_stream_open_ioctl_locked()
1948 * xe_oa_stream_open_ioctl - Opens an OA stream
1953 * The functions opens an OA stream. An OA stream, opened with specified
1989 * without global stream access, can be an unprivileged operation in xe_oa_stream_open_ioctl()
2004 drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n"); in xe_oa_stream_open_ioctl()
2272 * OA stream.
2543 /* Ensure MMIO trigger remains disabled till there is a stream */ in __xe_oa_init_oa_units()