143 static u32 xe_oa_circ_diff(struct xe_oa_stream *stream, u32 tail, u32 head)  in xe_oa_circ_diff()  argument
146 		tail + stream->oa_buffer.circ_size - head;  in xe_oa_circ_diff()
149 static u32 xe_oa_circ_incr(struct xe_oa_stream *stream, u32 ptr, u32 n)  in xe_oa_circ_incr()  argument
151 	return ptr + n >= stream->oa_buffer.circ_size ?  in xe_oa_circ_incr()
152 		ptr + n - stream->oa_buffer.circ_size : ptr + n;  in xe_oa_circ_incr()
198 static const struct xe_oa_regs *__oa_regs(struct xe_oa_stream *stream)  in __oa_regs()  argument
200 	return &stream->hwe->oa_unit->regs;  in __oa_regs()
203 static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream)  in xe_oa_hw_tail_read()  argument
205 	return xe_mmio_read32(&stream->gt->mmio, __oa_regs(stream)->oa_tail_ptr) &  in xe_oa_hw_tail_read()
212 static u64 oa_report_id(struct xe_oa_stream *stream, void *report)  in oa_report_id()  argument
214 	return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report;  in oa_report_id()
217 static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report)  in oa_report_id_clear()  argument
219 	if (oa_report_header_64bit(stream))  in oa_report_id_clear()
225 static u64 oa_timestamp(struct xe_oa_stream *stream, void *report)  in oa_timestamp()  argument
227 	return oa_report_header_64bit(stream) ?  in oa_timestamp()
232 static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report)  in oa_timestamp_clear()  argument
234 	if (oa_report_header_64bit(stream))  in oa_timestamp_clear()
240 static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream)  in xe_oa_buffer_check_unlocked()  argument
242 	u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);  in xe_oa_buffer_check_unlocked()
243 	int report_size = stream->oa_buffer.format->size;  in xe_oa_buffer_check_unlocked()
249 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in xe_oa_buffer_check_unlocked()
251 	hw_tail = xe_oa_hw_tail_read(stream);  in xe_oa_buffer_check_unlocked()
259 	partial_report_size = xe_oa_circ_diff(stream, hw_tail, stream->oa_buffer.tail);  in xe_oa_buffer_check_unlocked()
263 	hw_tail = xe_oa_circ_diff(stream, hw_tail, partial_report_size);  in xe_oa_buffer_check_unlocked()
275 	while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) {  in xe_oa_buffer_check_unlocked()
276 		void *report = stream->oa_buffer.vaddr + tail;  in xe_oa_buffer_check_unlocked()
278 		if (oa_report_id(stream, report) || oa_timestamp(stream, report))  in xe_oa_buffer_check_unlocked()
281 		tail = xe_oa_circ_diff(stream, tail, report_size);  in xe_oa_buffer_check_unlocked()
284 	if (xe_oa_circ_diff(stream, hw_tail, tail) > report_size)  in xe_oa_buffer_check_unlocked()
285 		drm_dbg(&stream->oa->xe->drm,  in xe_oa_buffer_check_unlocked()
287 			stream->oa_buffer.head, tail, hw_tail);  in xe_oa_buffer_check_unlocked()
289 	stream->oa_buffer.tail = tail;  in xe_oa_buffer_check_unlocked()
291 	pollin = xe_oa_circ_diff(stream, stream->oa_buffer.tail,  in xe_oa_buffer_check_unlocked()
292 				 stream->oa_buffer.head) >= report_size;  in xe_oa_buffer_check_unlocked()
294 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in xe_oa_buffer_check_unlocked()
301 	struct xe_oa_stream *stream =  in xe_oa_poll_check_timer_cb()  local
302 		container_of(hrtimer, typeof(*stream), poll_check_timer);  in xe_oa_poll_check_timer_cb()
304 	if (xe_oa_buffer_check_unlocked(stream)) {  in xe_oa_poll_check_timer_cb()
305 		stream->pollin = true;  in xe_oa_poll_check_timer_cb()
306 		wake_up(&stream->poll_wq);  in xe_oa_poll_check_timer_cb()
309 	hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period_ns));  in xe_oa_poll_check_timer_cb()
314 static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf,  in xe_oa_append_report()  argument
317 	int report_size = stream->oa_buffer.format->size;  in xe_oa_append_report()
326 	oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size;  in xe_oa_append_report()
334 		if (copy_to_user(buf, stream->oa_buffer.vaddr,  in xe_oa_append_report()
346 static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf,  in xe_oa_append_reports()  argument
349 	int report_size = stream->oa_buffer.format->size;  in xe_oa_append_reports()
350 	u8 *oa_buf_base = stream->oa_buffer.vaddr;  in xe_oa_append_reports()
351 	u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);  in xe_oa_append_reports()
357 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in xe_oa_append_reports()
358 	head = stream->oa_buffer.head;  in xe_oa_append_reports()
359 	tail = stream->oa_buffer.tail;  in xe_oa_append_reports()
360 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in xe_oa_append_reports()
362 	xe_assert(stream->oa->xe,  in xe_oa_append_reports()
363 		  head < stream->oa_buffer.circ_size && tail < stream->oa_buffer.circ_size);  in xe_oa_append_reports()
365 	for (; xe_oa_circ_diff(stream, tail, head);  in xe_oa_append_reports()
366 	     head = xe_oa_circ_incr(stream, head, report_size)) {  in xe_oa_append_reports()
369 		ret = xe_oa_append_report(stream, buf, count, offset, report);  in xe_oa_append_reports()
373 		if (!(stream->oa_buffer.circ_size % report_size)) {  in xe_oa_append_reports()
375 			oa_report_id_clear(stream, (void *)report);  in xe_oa_append_reports()
376 			oa_timestamp_clear(stream, (void *)report);  in xe_oa_append_reports()
378 			u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size;  in xe_oa_append_reports()
392 		struct xe_reg oaheadptr = __oa_regs(stream)->oa_head_ptr;  in xe_oa_append_reports()
394 		spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in xe_oa_append_reports()
395 		xe_mmio_write32(&stream->gt->mmio, oaheadptr,  in xe_oa_append_reports()
397 		stream->oa_buffer.head = head;  in xe_oa_append_reports()
398 		spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in xe_oa_append_reports()
404 static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream)  in xe_oa_init_oa_buffer()  argument
406 	struct xe_mmio *mmio = &stream->gt->mmio;  in xe_oa_init_oa_buffer()
407 	u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);  in xe_oa_init_oa_buffer()
411 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);  in xe_oa_init_oa_buffer()
413 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_status, 0);  in xe_oa_init_oa_buffer()
414 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_head_ptr,  in xe_oa_init_oa_buffer()
416 	stream->oa_buffer.head = 0;  in xe_oa_init_oa_buffer()
421 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_buffer, oa_buf);  in xe_oa_init_oa_buffer()
422 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_tail_ptr,  in xe_oa_init_oa_buffer()
426 	stream->oa_buffer.tail = 0;  in xe_oa_init_oa_buffer()
428 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);  in xe_oa_init_oa_buffer()
431 	memset(stream->oa_buffer.vaddr, 0, stream->oa_buffer.bo->size);  in xe_oa_init_oa_buffer()
441 static u32 __oa_ccs_select(struct xe_oa_stream *stream)  in __oa_ccs_select()  argument
445 	if (stream->hwe->class != XE_ENGINE_CLASS_COMPUTE)  in __oa_ccs_select()
448 	val = REG_FIELD_PREP(OAG_OACONTROL_OA_CCS_SELECT_MASK, stream->hwe->instance);  in __oa_ccs_select()
449 	xe_assert(stream->oa->xe,  in __oa_ccs_select()
450 		  REG_FIELD_GET(OAG_OACONTROL_OA_CCS_SELECT_MASK, val) == stream->hwe->instance);  in __oa_ccs_select()
454 static void xe_oa_enable(struct xe_oa_stream *stream)  in xe_oa_enable()  argument
456 	const struct xe_oa_format *format = stream->oa_buffer.format;  in xe_oa_enable()
464 	xe_oa_init_oa_buffer(stream);  in xe_oa_enable()
466 	regs = __oa_regs(stream);  in xe_oa_enable()
468 		__oa_ccs_select(stream) | OAG_OACONTROL_OA_COUNTER_ENABLE;  in xe_oa_enable()
470 	if (GRAPHICS_VER(stream->oa->xe) >= 20 &&  in xe_oa_enable()
471 	    stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG)  in xe_oa_enable()
474 	xe_mmio_write32(&stream->gt->mmio, regs->oa_ctrl, val);  in xe_oa_enable()
477 static void xe_oa_disable(struct xe_oa_stream *stream)  in xe_oa_disable()  argument
479 	struct xe_mmio *mmio = &stream->gt->mmio;  in xe_oa_disable()
481 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctrl, 0);  in xe_oa_disable()
482 	if (xe_mmio_wait32(mmio, __oa_regs(stream)->oa_ctrl,  in xe_oa_disable()
484 		drm_err(&stream->oa->xe->drm,  in xe_oa_disable()
487 	if (GRAPHICS_VERx100(stream->oa->xe) <= 1270 && GRAPHICS_VERx100(stream->oa->xe) != 1260) {  in xe_oa_disable()
491 			drm_err(&stream->oa->xe->drm,  in xe_oa_disable()
496 static int xe_oa_wait_unlocked(struct xe_oa_stream *stream)  in xe_oa_wait_unlocked()  argument
499 	if (!stream->periodic)  in xe_oa_wait_unlocked()
502 	return wait_event_interruptible(stream->poll_wq,  in xe_oa_wait_unlocked()
503 					xe_oa_buffer_check_unlocked(stream));  in xe_oa_wait_unlocked()
509 static int __xe_oa_read(struct xe_oa_stream *stream, char __user *buf,  in __xe_oa_read()  argument
513 	stream->oa_status = xe_mmio_rmw32(&stream->gt->mmio, __oa_regs(stream)->oa_status,  in __xe_oa_read()
519 	if (stream->oa_status & OASTATUS_RELEVANT_BITS)  in __xe_oa_read()
522 	return xe_oa_append_reports(stream, buf, count, offset);  in __xe_oa_read()
528 	struct xe_oa_stream *stream = file->private_data;  in xe_oa_read()  local
533 	if (!stream->enabled || !stream->sample)  in xe_oa_read()
538 			ret = xe_oa_wait_unlocked(stream);  in xe_oa_read()
542 			mutex_lock(&stream->stream_lock);  in xe_oa_read()
543 			ret = __xe_oa_read(stream, buf, count, &offset);  in xe_oa_read()
544 			mutex_unlock(&stream->stream_lock);  in xe_oa_read()
547 		mutex_lock(&stream->stream_lock);  in xe_oa_read()
548 		ret = __xe_oa_read(stream, buf, count, &offset);  in xe_oa_read()
549 		mutex_unlock(&stream->stream_lock);  in xe_oa_read()
562 		stream->pollin = false;  in xe_oa_read()
568 static __poll_t xe_oa_poll_locked(struct xe_oa_stream *stream,  in xe_oa_poll_locked()  argument
573 	poll_wait(file, &stream->poll_wq, wait);  in xe_oa_poll_locked()
581 	if (stream->pollin)  in xe_oa_poll_locked()
589 	struct xe_oa_stream *stream = file->private_data;  in xe_oa_poll()  local
592 	mutex_lock(&stream->stream_lock);  in xe_oa_poll()
593 	ret = xe_oa_poll_locked(stream, file, wait);  in xe_oa_poll()
594 	mutex_unlock(&stream->stream_lock);  in xe_oa_poll()
599 static struct dma_fence *xe_oa_submit_bb(struct xe_oa_stream *stream, enum xe_oa_submit_deps deps,  in xe_oa_submit_bb()  argument
607 	job = xe_bb_create_job(stream->k_exec_q, bb);  in xe_oa_submit_bb()
614 		for (int i = 0; i < stream->num_syncs && !err; i++)  in xe_oa_submit_bb()
615 			err = xe_sync_entry_add_deps(&stream->syncs[i], job);  in xe_oa_submit_bb()
617 			drm_dbg(&stream->oa->xe->drm, "xe_sync_entry_add_deps err %d\n", err);  in xe_oa_submit_bb()
663 static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream)  in xe_oa_free_oa_buffer()  argument
665 	xe_bo_unpin_map_no_vm(stream->oa_buffer.bo);  in xe_oa_free_oa_buffer()
668 static void xe_oa_free_configs(struct xe_oa_stream *stream)  in xe_oa_free_configs()  argument
672 	xe_oa_config_put(stream->oa_config);  in xe_oa_free_configs()
673 	llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)  in xe_oa_free_configs()
674 		free_oa_config_bo(oa_bo, stream->last_fence);  in xe_oa_free_configs()
675 	dma_fence_put(stream->last_fence);  in xe_oa_free_configs()
678 static void xe_oa_store_flex(struct xe_oa_stream *stream, struct xe_lrc *lrc,  in xe_oa_store_flex()  argument
692 static int xe_oa_modify_ctx_image(struct xe_oa_stream *stream, struct xe_lrc *lrc,  in xe_oa_modify_ctx_image()  argument
699 	bb = xe_bb_new(stream->gt, 4 * count, false);  in xe_oa_modify_ctx_image()
705 	xe_oa_store_flex(stream, lrc, bb, flex, count);  in xe_oa_modify_ctx_image()
707 	fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_NO_DEPS, bb);  in xe_oa_modify_ctx_image()
722 static int xe_oa_load_with_lri(struct xe_oa_stream *stream, struct xe_oa_reg *reg_lri)  in xe_oa_load_with_lri()  argument
728 	bb = xe_bb_new(stream->gt, 3, false);  in xe_oa_load_with_lri()
736 	fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_NO_DEPS, bb);  in xe_oa_load_with_lri()
751 static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable)  in xe_oa_configure_oar_context()  argument
753 	const struct xe_oa_format *format = stream->oa_buffer.format;  in xe_oa_configure_oar_context()
754 	struct xe_lrc *lrc = stream->exec_q->lrc[0];  in xe_oa_configure_oar_context()
761 			OACTXCONTROL(stream->hwe->mmio_base),  in xe_oa_configure_oar_context()
762 			stream->oa->ctx_oactxctrl_offset[stream->hwe->class] + 1,  in xe_oa_configure_oar_context()
766 			RING_CONTEXT_CONTROL(stream->hwe->mmio_base),  in xe_oa_configure_oar_context()
775 	err = xe_oa_modify_ctx_image(stream, stream->exec_q->lrc[0],  in xe_oa_configure_oar_context()
781 	return xe_oa_load_with_lri(stream, &reg_lri);  in xe_oa_configure_oar_context()
784 static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable)  in xe_oa_configure_oac_context()  argument
786 	const struct xe_oa_format *format = stream->oa_buffer.format;  in xe_oa_configure_oac_context()
787 	struct xe_lrc *lrc = stream->exec_q->lrc[0];  in xe_oa_configure_oac_context()
793 			OACTXCONTROL(stream->hwe->mmio_base),  in xe_oa_configure_oac_context()
794 			stream->oa->ctx_oactxctrl_offset[stream->hwe->class] + 1,  in xe_oa_configure_oac_context()
798 			RING_CONTEXT_CONTROL(stream->hwe->mmio_base),  in xe_oa_configure_oac_context()
808 	xe_mmio_write32(&stream->gt->mmio, __oa_regs(stream)->oa_ctrl,  in xe_oa_configure_oac_context()
809 			__oa_ccs_select(stream));  in xe_oa_configure_oac_context()
812 	err = xe_oa_modify_ctx_image(stream, stream->exec_q->lrc[0],  in xe_oa_configure_oac_context()
818 	return xe_oa_load_with_lri(stream, &reg_lri);  in xe_oa_configure_oac_context()
821 static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable)  in xe_oa_configure_oa_context()  argument
823 	switch (stream->hwe->class) {  in xe_oa_configure_oa_context()
825 		return xe_oa_configure_oar_context(stream, enable);  in xe_oa_configure_oa_context()
827 		return xe_oa_configure_oac_context(stream, enable);  in xe_oa_configure_oa_context()
836 static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable)  in oag_configure_mmio_trigger()  argument
839 			     enable && stream && stream->sample ?  in oag_configure_mmio_trigger()
843 static void xe_oa_disable_metric_set(struct xe_oa_stream *stream)  in xe_oa_disable_metric_set()  argument
845 	struct xe_mmio *mmio = &stream->gt->mmio;  in xe_oa_disable_metric_set()
852 	if (stream->oa->xe->info.platform == XE_DG2) {  in xe_oa_disable_metric_set()
853 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN,  in xe_oa_disable_metric_set()
855 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2,  in xe_oa_disable_metric_set()
859 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug,  in xe_oa_disable_metric_set()
860 			oag_configure_mmio_trigger(stream, false));  in xe_oa_disable_metric_set()
863 	if (stream->exec_q)  in xe_oa_disable_metric_set()
864 		xe_oa_configure_oa_context(stream, false);  in xe_oa_disable_metric_set()
870 		 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0);  in xe_oa_disable_metric_set()
876 static void xe_oa_stream_destroy(struct xe_oa_stream *stream)  in xe_oa_stream_destroy()  argument
878 	struct xe_oa_unit *u = stream->hwe->oa_unit;  in xe_oa_stream_destroy()
879 	struct xe_gt *gt = stream->hwe->gt;  in xe_oa_stream_destroy()
881 	if (WARN_ON(stream != u->exclusive_stream))  in xe_oa_stream_destroy()
886 	mutex_destroy(&stream->stream_lock);  in xe_oa_stream_destroy()
888 	xe_oa_disable_metric_set(stream);  in xe_oa_stream_destroy()
889 	xe_exec_queue_put(stream->k_exec_q);  in xe_oa_stream_destroy()
891 	xe_oa_free_oa_buffer(stream);  in xe_oa_stream_destroy()
894 	xe_pm_runtime_put(stream->oa->xe);  in xe_oa_stream_destroy()
897 	if (stream->override_gucrc)  in xe_oa_stream_destroy()
900 	xe_oa_free_configs(stream);  in xe_oa_stream_destroy()
901 	xe_file_put(stream->xef);  in xe_oa_stream_destroy()
904 static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream)  in xe_oa_alloc_oa_buffer()  argument
911 	bo = xe_bo_create_pin_map(stream->oa->xe, stream->gt->tile, NULL,  in xe_oa_alloc_oa_buffer()
917 	stream->oa_buffer.bo = bo;  in xe_oa_alloc_oa_buffer()
919 	xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0);  in xe_oa_alloc_oa_buffer()
920 	stream->oa_buffer.vaddr = bo->vmap.vaddr;  in xe_oa_alloc_oa_buffer()
925 __xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config)  in __xe_oa_alloc_config_buffer()  argument
938 	bb = xe_bb_new(stream->gt, config_length, false);  in __xe_oa_alloc_config_buffer()
946 	llist_add(&oa_bo->node, &stream->oa_config_bos);  in __xe_oa_alloc_config_buffer()
955 xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config)  in xe_oa_alloc_config_buffer()  argument
960 	llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {  in xe_oa_alloc_config_buffer()
967 	oa_bo = __xe_oa_alloc_config_buffer(stream, oa_config);  in xe_oa_alloc_config_buffer()
972 static void xe_oa_update_last_fence(struct xe_oa_stream *stream, struct dma_fence *fence)  in xe_oa_update_last_fence()  argument
974 	dma_fence_put(stream->last_fence);  in xe_oa_update_last_fence()
975 	stream->last_fence = dma_fence_get(fence);  in xe_oa_update_last_fence()
1015 static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config *config)  in xe_oa_emit_oa_config()  argument
1029 	oa_bo = xe_oa_alloc_config_buffer(stream, config);  in xe_oa_emit_oa_config()
1036 	fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_ADD_DEPS, oa_bo->bb);  in xe_oa_emit_oa_config()
1046 	for (i = 0; i < stream->num_syncs; i++) {  in xe_oa_emit_oa_config()
1047 		if (stream->syncs[i].flags & DRM_XE_SYNC_FLAG_SIGNAL)  in xe_oa_emit_oa_config()
1049 		xe_sync_entry_signal(&stream->syncs[i], &ofence->base);  in xe_oa_emit_oa_config()
1057 	xe_oa_update_last_fence(stream, fence);  in xe_oa_emit_oa_config()
1061 	xe_gt_assert(stream->gt, !err || err == -ENOENT);  in xe_oa_emit_oa_config()
1072 	for (i = 0; i < stream->num_syncs; i++)  in xe_oa_emit_oa_config()
1073 		xe_sync_entry_cleanup(&stream->syncs[i]);  in xe_oa_emit_oa_config()
1074 	kfree(stream->syncs);  in xe_oa_emit_oa_config()
1082 static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream)  in oag_report_ctx_switches()  argument
1086 			     stream->sample ?  in oag_report_ctx_switches()
1090 static int xe_oa_enable_metric_set(struct xe_oa_stream *stream)  in xe_oa_enable_metric_set()  argument
1092 	struct xe_mmio *mmio = &stream->gt->mmio;  in xe_oa_enable_metric_set()
1101 	if (stream->oa->xe->info.platform == XE_DG2) {  in xe_oa_enable_metric_set()
1102 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN,  in xe_oa_enable_metric_set()
1104 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2,  in xe_oa_enable_metric_set()
1112 	if (GRAPHICS_VER(stream->oa->xe) >= 20)  in xe_oa_enable_metric_set()
1119 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug,  in xe_oa_enable_metric_set()
1121 			oag_report_ctx_switches(stream) |  in xe_oa_enable_metric_set()
1122 			oag_configure_mmio_trigger(stream, true));  in xe_oa_enable_metric_set()
1124 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ?  in xe_oa_enable_metric_set()
1128 					stream->period_exponent)) : 0);  in xe_oa_enable_metric_set()
1136 		 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0);  in xe_oa_enable_metric_set()
1141 	if (stream->exec_q) {  in xe_oa_enable_metric_set()
1142 		ret = xe_oa_configure_oa_context(stream, true);  in xe_oa_enable_metric_set()
1147 	return xe_oa_emit_oa_config(stream, stream->oa_config);  in xe_oa_enable_metric_set()
1406 static void xe_oa_stream_enable(struct xe_oa_stream *stream)  in xe_oa_stream_enable()  argument
1408 	stream->pollin = false;  in xe_oa_stream_enable()
1410 	xe_oa_enable(stream);  in xe_oa_stream_enable()
1412 	if (stream->sample)  in xe_oa_stream_enable()
1413 		hrtimer_start(&stream->poll_check_timer,  in xe_oa_stream_enable()
1414 			      ns_to_ktime(stream->poll_period_ns),  in xe_oa_stream_enable()
1418 static void xe_oa_stream_disable(struct xe_oa_stream *stream)  in xe_oa_stream_disable()  argument
1420 	xe_oa_disable(stream);  in xe_oa_stream_disable()
1422 	if (stream->sample)  in xe_oa_stream_disable()
1423 		hrtimer_cancel(&stream->poll_check_timer);  in xe_oa_stream_disable()
1426 static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream)  in xe_oa_enable_preempt_timeslice()  argument
1428 	struct xe_exec_queue *q = stream->exec_q;  in xe_oa_enable_preempt_timeslice()
1432 	ret1 = q->ops->set_timeslice(q, stream->hwe->eclass->sched_props.timeslice_us);  in xe_oa_enable_preempt_timeslice()
1433 	ret2 = q->ops->set_preempt_timeout(q, stream->hwe->eclass->sched_props.preempt_timeout_us);  in xe_oa_enable_preempt_timeslice()
1438 	drm_dbg(&stream->oa->xe->drm, "%s failed ret1 %d ret2 %d\n", __func__, ret1, ret2);  in xe_oa_enable_preempt_timeslice()
1442 static int xe_oa_disable_preempt_timeslice(struct xe_oa_stream *stream)  in xe_oa_disable_preempt_timeslice()  argument
1444 	struct xe_exec_queue *q = stream->exec_q;  in xe_oa_disable_preempt_timeslice()
1458 	xe_oa_enable_preempt_timeslice(stream);  in xe_oa_disable_preempt_timeslice()
1459 	drm_dbg(&stream->oa->xe->drm, "%s failed %d\n", __func__, ret);  in xe_oa_disable_preempt_timeslice()
1463 static int xe_oa_enable_locked(struct xe_oa_stream *stream)  in xe_oa_enable_locked()  argument
1465 	if (stream->enabled)  in xe_oa_enable_locked()
1468 	if (stream->no_preempt) {  in xe_oa_enable_locked()
1469 		int ret = xe_oa_disable_preempt_timeslice(stream);  in xe_oa_enable_locked()
1475 	xe_oa_stream_enable(stream);  in xe_oa_enable_locked()
1477 	stream->enabled = true;  in xe_oa_enable_locked()
1481 static int xe_oa_disable_locked(struct xe_oa_stream *stream)  in xe_oa_disable_locked()  argument
1485 	if (!stream->enabled)  in xe_oa_disable_locked()
1488 	xe_oa_stream_disable(stream);  in xe_oa_disable_locked()
1490 	if (stream->no_preempt)  in xe_oa_disable_locked()
1491 		ret = xe_oa_enable_preempt_timeslice(stream);  in xe_oa_disable_locked()
1493 	stream->enabled = false;  in xe_oa_disable_locked()
1497 static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg)  in xe_oa_config_locked()  argument
1500 	long ret = stream->oa_config->id;  in xe_oa_config_locked()
1504 	err = xe_oa_user_extensions(stream->oa, XE_OA_USER_EXTN_FROM_CONFIG, arg, 0, &param);  in xe_oa_config_locked()
1508 	config = xe_oa_get_oa_config(stream->oa, param.metric_set);  in xe_oa_config_locked()
1512 	param.xef = stream->xef;  in xe_oa_config_locked()
1513 	err = xe_oa_parse_syncs(stream->oa, &param);  in xe_oa_config_locked()
1517 	stream->num_syncs = param.num_syncs;  in xe_oa_config_locked()
1518 	stream->syncs = param.syncs;  in xe_oa_config_locked()
1520 	err = xe_oa_emit_oa_config(stream, config);  in xe_oa_config_locked()
1522 		config = xchg(&stream->oa_config, config);  in xe_oa_config_locked()
1523 		drm_dbg(&stream->oa->xe->drm, "changed to oa config uuid=%s\n",  in xe_oa_config_locked()
1524 			stream->oa_config->uuid);  in xe_oa_config_locked()
1533 static long xe_oa_status_locked(struct xe_oa_stream *stream, unsigned long arg)  in xe_oa_status_locked()  argument
1539 	if (stream->oa_status & OASTATUS_REPORT_LOST)  in xe_oa_status_locked()
1541 	if (stream->oa_status & OASTATUS_BUFFER_OVERFLOW)  in xe_oa_status_locked()
1543 	if (stream->oa_status & OASTATUS_COUNTER_OVERFLOW)  in xe_oa_status_locked()
1545 	if (stream->oa_status & OASTATUS_MMIO_TRG_Q_FULL)  in xe_oa_status_locked()
1554 static long xe_oa_info_locked(struct xe_oa_stream *stream, unsigned long arg)  in xe_oa_info_locked()  argument
1565 static long xe_oa_ioctl_locked(struct xe_oa_stream *stream,  in xe_oa_ioctl_locked()  argument
1571 		return xe_oa_enable_locked(stream);  in xe_oa_ioctl_locked()
1573 		return xe_oa_disable_locked(stream);  in xe_oa_ioctl_locked()
1575 		return xe_oa_config_locked(stream, arg);  in xe_oa_ioctl_locked()
1577 		return xe_oa_status_locked(stream, arg);  in xe_oa_ioctl_locked()
1579 		return xe_oa_info_locked(stream, arg);  in xe_oa_ioctl_locked()
1589 	struct xe_oa_stream *stream = file->private_data;  in xe_oa_ioctl()  local
1592 	mutex_lock(&stream->stream_lock);  in xe_oa_ioctl()
1593 	ret = xe_oa_ioctl_locked(stream, cmd, arg);  in xe_oa_ioctl()
1594 	mutex_unlock(&stream->stream_lock);  in xe_oa_ioctl()
1599 static void xe_oa_destroy_locked(struct xe_oa_stream *stream)  in xe_oa_destroy_locked()  argument
1601 	if (stream->enabled)  in xe_oa_destroy_locked()
1602 		xe_oa_disable_locked(stream);  in xe_oa_destroy_locked()
1604 	xe_oa_stream_destroy(stream);  in xe_oa_destroy_locked()
1606 	if (stream->exec_q)  in xe_oa_destroy_locked()
1607 		xe_exec_queue_put(stream->exec_q);  in xe_oa_destroy_locked()
1609 	kfree(stream);  in xe_oa_destroy_locked()
1614 	struct xe_oa_stream *stream = file->private_data;  in xe_oa_release()  local
1615 	struct xe_gt *gt = stream->gt;  in xe_oa_release()
1619 	xe_oa_destroy_locked(stream);  in xe_oa_release()
1631 	struct xe_oa_stream *stream = file->private_data;  in xe_oa_mmap()  local
1632 	struct xe_bo *bo = stream->oa_buffer.bo;  in xe_oa_mmap()
1637 		drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n");  in xe_oa_mmap()
1643 		drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n");  in xe_oa_mmap()
1652 		drm_dbg(&stream->oa->xe->drm, "mmap must be read only\n");  in xe_oa_mmap()
1658 	xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == vma_pages(vma));  in xe_oa_mmap()
1707 static u32 xe_oa_context_image_offset(struct xe_oa_stream *stream, u32 reg)  in xe_oa_context_image_offset()  argument
1709 	struct xe_lrc *lrc = stream->exec_q->lrc[0];  in xe_oa_context_image_offset()
1710 	u32 len = (xe_gt_lrc_size(stream->gt, stream->hwe->class) +  in xe_oa_context_image_offset()
1715 	if (drm_WARN_ON(&stream->oa->xe->drm, !state))  in xe_oa_context_image_offset()
1724 			drm_WARN_ON(&stream->oa->xe->drm,  in xe_oa_context_image_offset()
1737 static int xe_oa_set_ctx_ctrl_offset(struct xe_oa_stream *stream)  in xe_oa_set_ctx_ctrl_offset()  argument
1739 	struct xe_reg reg = OACTXCONTROL(stream->hwe->mmio_base);  in xe_oa_set_ctx_ctrl_offset()
1740 	u32 offset = stream->oa->ctx_oactxctrl_offset[stream->hwe->class];  in xe_oa_set_ctx_ctrl_offset()
1746 	offset = xe_oa_context_image_offset(stream, reg.addr);  in xe_oa_set_ctx_ctrl_offset()
1747 	stream->oa->ctx_oactxctrl_offset[stream->hwe->class] = offset;  in xe_oa_set_ctx_ctrl_offset()
1749 	drm_dbg(&stream->oa->xe->drm, "%s oa ctx control at 0x%08x dword offset\n",  in xe_oa_set_ctx_ctrl_offset()
1750 		stream->hwe->name, offset);  in xe_oa_set_ctx_ctrl_offset()
1755 static int xe_oa_stream_init(struct xe_oa_stream *stream,  in xe_oa_stream_init()  argument
1763 	stream->exec_q = param->exec_q;  in xe_oa_stream_init()
1764 	stream->poll_period_ns = DEFAULT_POLL_PERIOD_NS;  in xe_oa_stream_init()
1765 	stream->hwe = param->hwe;  in xe_oa_stream_init()
1766 	stream->gt = stream->hwe->gt;  in xe_oa_stream_init()
1767 	stream->oa_buffer.format = &stream->oa->oa_formats[param->oa_format];  in xe_oa_stream_init()
1769 	stream->sample = param->sample;  in xe_oa_stream_init()
1770 	stream->periodic = param->period_exponent > 0;  in xe_oa_stream_init()
1771 	stream->period_exponent = param->period_exponent;  in xe_oa_stream_init()
1772 	stream->no_preempt = param->no_preempt;  in xe_oa_stream_init()
1774 	stream->xef = xe_file_get(param->xef);  in xe_oa_stream_init()
1775 	stream->num_syncs = param->num_syncs;  in xe_oa_stream_init()
1776 	stream->syncs = param->syncs;  in xe_oa_stream_init()
1783 	if (GRAPHICS_VER(stream->oa->xe) >= 20 &&  in xe_oa_stream_init()
1784 	    stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG && stream->sample)  in xe_oa_stream_init()
1785 		stream->oa_buffer.circ_size =  in xe_oa_stream_init()
1786 			XE_OA_BUFFER_SIZE - XE_OA_BUFFER_SIZE % stream->oa_buffer.format->size;  in xe_oa_stream_init()
1788 		stream->oa_buffer.circ_size = XE_OA_BUFFER_SIZE;  in xe_oa_stream_init()
1790 	if (stream->exec_q && engine_supports_mi_query(stream->hwe)) {  in xe_oa_stream_init()
1792 		ret = xe_oa_set_ctx_ctrl_offset(stream);  in xe_oa_stream_init()
1794 			drm_err(&stream->oa->xe->drm,  in xe_oa_stream_init()
1796 				stream->hwe->name);  in xe_oa_stream_init()
1801 	stream->oa_config = xe_oa_get_oa_config(stream->oa, param->metric_set);  in xe_oa_stream_init()
1802 	if (!stream->oa_config) {  in xe_oa_stream_init()
1803 		drm_dbg(&stream->oa->xe->drm, "Invalid OA config id=%i\n", param->metric_set);  in xe_oa_stream_init()
1814 	if (stream->oa->xe->info.platform == XE_PVC) {  in xe_oa_stream_init()
1820 		stream->override_gucrc = true;  in xe_oa_stream_init()
1824 	xe_pm_runtime_get(stream->oa->xe);  in xe_oa_stream_init()
1831 	ret = xe_oa_alloc_oa_buffer(stream);  in xe_oa_stream_init()
1835 	stream->k_exec_q = xe_exec_queue_create(stream->oa->xe, NULL,  in xe_oa_stream_init()
1836 						BIT(stream->hwe->logical_instance), 1,  in xe_oa_stream_init()
1837 						stream->hwe, EXEC_QUEUE_FLAG_KERNEL, 0);  in xe_oa_stream_init()
1838 	if (IS_ERR(stream->k_exec_q)) {  in xe_oa_stream_init()
1839 		ret = PTR_ERR(stream->k_exec_q);  in xe_oa_stream_init()
1840 		drm_err(&stream->oa->xe->drm, "gt%d, hwe %s, xe_exec_queue_create failed=%d",  in xe_oa_stream_init()
1841 			stream->gt->info.id, stream->hwe->name, ret);  in xe_oa_stream_init()
1845 	ret = xe_oa_enable_metric_set(stream);  in xe_oa_stream_init()
1847 		drm_dbg(&stream->oa->xe->drm, "Unable to enable metric set\n");  in xe_oa_stream_init()
1851 	drm_dbg(&stream->oa->xe->drm, "opening stream oa config uuid=%s\n",  in xe_oa_stream_init()
1852 		stream->oa_config->uuid);  in xe_oa_stream_init()
1854 	WRITE_ONCE(u->exclusive_stream, stream);  in xe_oa_stream_init()
1856 	hrtimer_init(&stream->poll_check_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in xe_oa_stream_init()
1857 	stream->poll_check_timer.function = xe_oa_poll_check_timer_cb;  in xe_oa_stream_init()
1858 	init_waitqueue_head(&stream->poll_wq);  in xe_oa_stream_init()
1860 	spin_lock_init(&stream->oa_buffer.ptr_lock);  in xe_oa_stream_init()
1861 	mutex_init(&stream->stream_lock);  in xe_oa_stream_init()
1866 	xe_oa_disable_metric_set(stream);  in xe_oa_stream_init()
1867 	xe_exec_queue_put(stream->k_exec_q);  in xe_oa_stream_init()
1869 	xe_oa_free_oa_buffer(stream);  in xe_oa_stream_init()
1872 	xe_pm_runtime_put(stream->oa->xe);  in xe_oa_stream_init()
1873 	if (stream->override_gucrc)  in xe_oa_stream_init()
1876 	xe_oa_free_configs(stream);  in xe_oa_stream_init()
1878 	xe_file_put(stream->xef);  in xe_oa_stream_init()
1885 	struct xe_oa_stream *stream;  in xe_oa_stream_open_ioctl_locked()  local
1896 	stream = kzalloc(sizeof(*stream), GFP_KERNEL);  in xe_oa_stream_open_ioctl_locked()
1897 	if (!stream) {  in xe_oa_stream_open_ioctl_locked()
1902 	stream->oa = oa;  in xe_oa_stream_open_ioctl_locked()
1903 	ret = xe_oa_stream_init(stream, param);  in xe_oa_stream_open_ioctl_locked()
1908 		ret = xe_oa_enable_locked(stream);  in xe_oa_stream_open_ioctl_locked()
1913 	stream_fd = anon_inode_getfd("[xe_oa]", &xe_oa_fops, stream, 0);  in xe_oa_stream_open_ioctl_locked()
1920 	drm_dev_get(&stream->oa->xe->drm);  in xe_oa_stream_open_ioctl_locked()
1925 		xe_oa_disable_locked(stream);  in xe_oa_stream_open_ioctl_locked()
1927 	xe_oa_stream_destroy(stream);  in xe_oa_stream_open_ioctl_locked()
1929 	kfree(stream);  in xe_oa_stream_open_ioctl_locked()