xref: /linux/drivers/gpu/drm/xe/xe_oa.c (revision 72ef07554c5dcabb0053a147c4fd221a8e39bcfd)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2023-2024 Intel Corporation
4  */
5 
6 #include <linux/anon_inodes.h>
7 #include <linux/delay.h>
8 #include <linux/nospec.h>
9 #include <linux/poll.h>
10 
11 #include <drm/drm_drv.h>
12 #include <drm/drm_managed.h>
13 #include <drm/xe_drm.h>
14 
15 #include "abi/guc_actions_slpc_abi.h"
16 #include "instructions/xe_mi_commands.h"
17 #include "regs/xe_engine_regs.h"
18 #include "regs/xe_gt_regs.h"
19 #include "regs/xe_lrc_layout.h"
20 #include "regs/xe_oa_regs.h"
21 #include "xe_assert.h"
22 #include "xe_bb.h"
23 #include "xe_bo.h"
24 #include "xe_device.h"
25 #include "xe_exec_queue.h"
26 #include "xe_force_wake.h"
27 #include "xe_gt.h"
28 #include "xe_gt_mcr.h"
29 #include "xe_gt_printk.h"
30 #include "xe_guc_pc.h"
31 #include "xe_lrc.h"
32 #include "xe_macros.h"
33 #include "xe_mmio.h"
34 #include "xe_oa.h"
35 #include "xe_observation.h"
36 #include "xe_pm.h"
37 #include "xe_sched_job.h"
38 #include "xe_sriov.h"
39 
40 #define DEFAULT_POLL_FREQUENCY_HZ 200
41 #define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ)
42 #define XE_OA_UNIT_INVALID U32_MAX
43 
44 struct xe_oa_reg {
45 	struct xe_reg addr;
46 	u32 value;
47 };
48 
49 struct xe_oa_config {
50 	struct xe_oa *oa;
51 
52 	char uuid[UUID_STRING_LEN + 1];
53 	int id;
54 
55 	const struct xe_oa_reg *regs;
56 	u32 regs_len;
57 
58 	struct attribute_group sysfs_metric;
59 	struct attribute *attrs[2];
60 	struct kobj_attribute sysfs_metric_id;
61 
62 	struct kref ref;
63 	struct rcu_head rcu;
64 };
65 
66 struct flex {
67 	struct xe_reg reg;
68 	u32 offset;
69 	u32 value;
70 };
71 
72 struct xe_oa_open_param {
73 	u32 oa_unit_id;
74 	bool sample;
75 	u32 metric_set;
76 	enum xe_oa_format_name oa_format;
77 	int period_exponent;
78 	bool disabled;
79 	int exec_queue_id;
80 	int engine_instance;
81 	struct xe_exec_queue *exec_q;
82 	struct xe_hw_engine *hwe;
83 	bool no_preempt;
84 };
85 
86 struct xe_oa_config_bo {
87 	struct llist_node node;
88 
89 	struct xe_oa_config *oa_config;
90 	struct xe_bb *bb;
91 };
92 
93 #define DRM_FMT(x) DRM_XE_OA_FMT_TYPE_##x
94 
95 static const struct xe_oa_format oa_formats[] = {
96 	[XE_OA_FORMAT_C4_B8]			= { 7, 64,  DRM_FMT(OAG) },
97 	[XE_OA_FORMAT_A12]			= { 0, 64,  DRM_FMT(OAG) },
98 	[XE_OA_FORMAT_A12_B8_C8]		= { 2, 128, DRM_FMT(OAG) },
99 	[XE_OA_FORMAT_A32u40_A4u32_B8_C8]	= { 5, 256, DRM_FMT(OAG) },
100 	[XE_OAR_FORMAT_A32u40_A4u32_B8_C8]	= { 5, 256, DRM_FMT(OAR) },
101 	[XE_OA_FORMAT_A24u40_A14u32_B8_C8]	= { 5, 256, DRM_FMT(OAG) },
102 	[XE_OAC_FORMAT_A24u64_B8_C8]		= { 1, 320, DRM_FMT(OAC), HDR_64_BIT },
103 	[XE_OAC_FORMAT_A22u32_R2u32_B8_C8]	= { 2, 192, DRM_FMT(OAC), HDR_64_BIT },
104 	[XE_OAM_FORMAT_MPEC8u64_B8_C8]		= { 1, 192, DRM_FMT(OAM_MPEC), HDR_64_BIT },
105 	[XE_OAM_FORMAT_MPEC8u32_B8_C8]		= { 2, 128, DRM_FMT(OAM_MPEC), HDR_64_BIT },
106 	[XE_OA_FORMAT_PEC64u64]			= { 1, 576, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
107 	[XE_OA_FORMAT_PEC64u64_B8_C8]		= { 1, 640, DRM_FMT(PEC), HDR_64_BIT, 1, 1 },
108 	[XE_OA_FORMAT_PEC64u32]			= { 1, 320, DRM_FMT(PEC), HDR_64_BIT },
109 	[XE_OA_FORMAT_PEC32u64_G1]		= { 5, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
110 	[XE_OA_FORMAT_PEC32u32_G1]		= { 5, 192, DRM_FMT(PEC), HDR_64_BIT },
111 	[XE_OA_FORMAT_PEC32u64_G2]		= { 6, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
112 	[XE_OA_FORMAT_PEC32u32_G2]		= { 6, 192, DRM_FMT(PEC), HDR_64_BIT },
113 	[XE_OA_FORMAT_PEC36u64_G1_32_G2_4]	= { 3, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
114 	[XE_OA_FORMAT_PEC36u64_G1_4_G2_32]	= { 4, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
115 };
116 
117 static u32 xe_oa_circ_diff(struct xe_oa_stream *stream, u32 tail, u32 head)
118 {
119 	return tail >= head ? tail - head :
120 		tail + stream->oa_buffer.circ_size - head;
121 }
122 
123 static u32 xe_oa_circ_incr(struct xe_oa_stream *stream, u32 ptr, u32 n)
124 {
125 	return ptr + n >= stream->oa_buffer.circ_size ?
126 		ptr + n - stream->oa_buffer.circ_size : ptr + n;
127 }
128 
129 static void xe_oa_config_release(struct kref *ref)
130 {
131 	struct xe_oa_config *oa_config =
132 		container_of(ref, typeof(*oa_config), ref);
133 
134 	kfree(oa_config->regs);
135 
136 	kfree_rcu(oa_config, rcu);
137 }
138 
139 static void xe_oa_config_put(struct xe_oa_config *oa_config)
140 {
141 	if (!oa_config)
142 		return;
143 
144 	kref_put(&oa_config->ref, xe_oa_config_release);
145 }
146 
147 static struct xe_oa_config *xe_oa_config_get(struct xe_oa_config *oa_config)
148 {
149 	return kref_get_unless_zero(&oa_config->ref) ? oa_config : NULL;
150 }
151 
152 static struct xe_oa_config *xe_oa_get_oa_config(struct xe_oa *oa, int metrics_set)
153 {
154 	struct xe_oa_config *oa_config;
155 
156 	rcu_read_lock();
157 	oa_config = idr_find(&oa->metrics_idr, metrics_set);
158 	if (oa_config)
159 		oa_config = xe_oa_config_get(oa_config);
160 	rcu_read_unlock();
161 
162 	return oa_config;
163 }
164 
165 static void free_oa_config_bo(struct xe_oa_config_bo *oa_bo)
166 {
167 	xe_oa_config_put(oa_bo->oa_config);
168 	xe_bb_free(oa_bo->bb, NULL);
169 	kfree(oa_bo);
170 }
171 
172 static const struct xe_oa_regs *__oa_regs(struct xe_oa_stream *stream)
173 {
174 	return &stream->hwe->oa_unit->regs;
175 }
176 
177 static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream)
178 {
179 	return xe_mmio_read32(stream->gt, __oa_regs(stream)->oa_tail_ptr) &
180 		OAG_OATAILPTR_MASK;
181 }
182 
183 #define oa_report_header_64bit(__s) \
184 	((__s)->oa_buffer.format->header == HDR_64_BIT)
185 
186 static u64 oa_report_id(struct xe_oa_stream *stream, void *report)
187 {
188 	return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report;
189 }
190 
191 static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report)
192 {
193 	if (oa_report_header_64bit(stream))
194 		*(u64 *)report = 0;
195 	else
196 		*report = 0;
197 }
198 
199 static u64 oa_timestamp(struct xe_oa_stream *stream, void *report)
200 {
201 	return oa_report_header_64bit(stream) ?
202 		*((u64 *)report + 1) :
203 		*((u32 *)report + 1);
204 }
205 
206 static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report)
207 {
208 	if (oa_report_header_64bit(stream))
209 		*(u64 *)&report[2] = 0;
210 	else
211 		report[1] = 0;
212 }
213 
214 static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream)
215 {
216 	u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);
217 	int report_size = stream->oa_buffer.format->size;
218 	u32 tail, hw_tail;
219 	unsigned long flags;
220 	bool pollin;
221 	u32 partial_report_size;
222 
223 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
224 
225 	hw_tail = xe_oa_hw_tail_read(stream);
226 	hw_tail -= gtt_offset;
227 
228 	/*
229 	 * The tail pointer increases in 64 byte (cacheline size), not in report_size
230 	 * increments. Also report size may not be a power of 2. Compute potential
231 	 * partially landed report in OA buffer.
232 	 */
233 	partial_report_size = xe_oa_circ_diff(stream, hw_tail, stream->oa_buffer.tail);
234 	partial_report_size %= report_size;
235 
236 	/* Subtract partial amount off the tail */
237 	hw_tail = xe_oa_circ_diff(stream, hw_tail, partial_report_size);
238 
239 	tail = hw_tail;
240 
241 	/*
242 	 * Walk the stream backward until we find a report with report id and timestamp
243 	 * not 0. We can't tell whether a report has fully landed in memory before the
244 	 * report id and timestamp of the following report have landed.
245 	 *
246 	 * This is assuming that the writes of the OA unit land in memory in the order
247 	 * they were written.  If not : (╯°□°)╯︵ ┻━┻
248 	 */
249 	while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) {
250 		void *report = stream->oa_buffer.vaddr + tail;
251 
252 		if (oa_report_id(stream, report) || oa_timestamp(stream, report))
253 			break;
254 
255 		tail = xe_oa_circ_diff(stream, tail, report_size);
256 	}
257 
258 	if (xe_oa_circ_diff(stream, hw_tail, tail) > report_size)
259 		drm_dbg(&stream->oa->xe->drm,
260 			"unlanded report(s) head=0x%x tail=0x%x hw_tail=0x%x\n",
261 			stream->oa_buffer.head, tail, hw_tail);
262 
263 	stream->oa_buffer.tail = tail;
264 
265 	pollin = xe_oa_circ_diff(stream, stream->oa_buffer.tail,
266 				 stream->oa_buffer.head) >= report_size;
267 
268 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
269 
270 	return pollin;
271 }
272 
273 static enum hrtimer_restart xe_oa_poll_check_timer_cb(struct hrtimer *hrtimer)
274 {
275 	struct xe_oa_stream *stream =
276 		container_of(hrtimer, typeof(*stream), poll_check_timer);
277 
278 	if (xe_oa_buffer_check_unlocked(stream)) {
279 		stream->pollin = true;
280 		wake_up(&stream->poll_wq);
281 	}
282 
283 	hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period_ns));
284 
285 	return HRTIMER_RESTART;
286 }
287 
288 static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf,
289 			       size_t count, size_t *offset, const u8 *report)
290 {
291 	int report_size = stream->oa_buffer.format->size;
292 	int report_size_partial;
293 	u8 *oa_buf_end;
294 
295 	if ((count - *offset) < report_size)
296 		return -ENOSPC;
297 
298 	buf += *offset;
299 
300 	oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size;
301 	report_size_partial = oa_buf_end - report;
302 
303 	if (report_size_partial < report_size) {
304 		if (copy_to_user(buf, report, report_size_partial))
305 			return -EFAULT;
306 		buf += report_size_partial;
307 
308 		if (copy_to_user(buf, stream->oa_buffer.vaddr,
309 				 report_size - report_size_partial))
310 			return -EFAULT;
311 	} else if (copy_to_user(buf, report, report_size)) {
312 		return -EFAULT;
313 	}
314 
315 	*offset += report_size;
316 
317 	return 0;
318 }
319 
320 static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf,
321 				size_t count, size_t *offset)
322 {
323 	int report_size = stream->oa_buffer.format->size;
324 	u8 *oa_buf_base = stream->oa_buffer.vaddr;
325 	u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);
326 	size_t start_offset = *offset;
327 	unsigned long flags;
328 	u32 head, tail;
329 	int ret = 0;
330 
331 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
332 	head = stream->oa_buffer.head;
333 	tail = stream->oa_buffer.tail;
334 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
335 
336 	xe_assert(stream->oa->xe,
337 		  head < stream->oa_buffer.circ_size && tail < stream->oa_buffer.circ_size);
338 
339 	for (; xe_oa_circ_diff(stream, tail, head);
340 	     head = xe_oa_circ_incr(stream, head, report_size)) {
341 		u8 *report = oa_buf_base + head;
342 
343 		ret = xe_oa_append_report(stream, buf, count, offset, report);
344 		if (ret)
345 			break;
346 
347 		if (!(stream->oa_buffer.circ_size % report_size)) {
348 			/* Clear out report id and timestamp to detect unlanded reports */
349 			oa_report_id_clear(stream, (void *)report);
350 			oa_timestamp_clear(stream, (void *)report);
351 		} else {
352 			u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size;
353 			u32 part = oa_buf_end - report;
354 
355 			/* Zero out the entire report */
356 			if (report_size <= part) {
357 				memset(report, 0, report_size);
358 			} else {
359 				memset(report, 0, part);
360 				memset(oa_buf_base, 0, report_size - part);
361 			}
362 		}
363 	}
364 
365 	if (start_offset != *offset) {
366 		struct xe_reg oaheadptr = __oa_regs(stream)->oa_head_ptr;
367 
368 		spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
369 		xe_mmio_write32(stream->gt, oaheadptr,
370 				(head + gtt_offset) & OAG_OAHEADPTR_MASK);
371 		stream->oa_buffer.head = head;
372 		spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
373 	}
374 
375 	return ret;
376 }
377 
378 static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream)
379 {
380 	u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);
381 	u32 oa_buf = gtt_offset | OABUFFER_SIZE_16M | OAG_OABUFFER_MEMORY_SELECT;
382 	unsigned long flags;
383 
384 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
385 
386 	xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_status, 0);
387 	xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_head_ptr,
388 			gtt_offset & OAG_OAHEADPTR_MASK);
389 	stream->oa_buffer.head = 0;
390 	/*
391 	 * PRM says: "This MMIO must be set before the OATAILPTR register and after the
392 	 * OAHEADPTR register. This is to enable proper functionality of the overflow bit".
393 	 */
394 	xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_buffer, oa_buf);
395 	xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_tail_ptr,
396 			gtt_offset & OAG_OATAILPTR_MASK);
397 
398 	/* Mark that we need updated tail pointer to read from */
399 	stream->oa_buffer.tail = 0;
400 
401 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
402 
403 	/* Zero out the OA buffer since we rely on zero report id and timestamp fields */
404 	memset(stream->oa_buffer.vaddr, 0, stream->oa_buffer.bo->size);
405 }
406 
407 static u32 __format_to_oactrl(const struct xe_oa_format *format, int counter_sel_mask)
408 {
409 	return ((format->counter_select << (ffs(counter_sel_mask) - 1)) & counter_sel_mask) |
410 		REG_FIELD_PREP(OA_OACONTROL_REPORT_BC_MASK, format->bc_report) |
411 		REG_FIELD_PREP(OA_OACONTROL_COUNTER_SIZE_MASK, format->counter_size);
412 }
413 
414 static u32 __oa_ccs_select(struct xe_oa_stream *stream)
415 {
416 	u32 val;
417 
418 	if (stream->hwe->class != XE_ENGINE_CLASS_COMPUTE)
419 		return 0;
420 
421 	val = REG_FIELD_PREP(OAG_OACONTROL_OA_CCS_SELECT_MASK, stream->hwe->instance);
422 	xe_assert(stream->oa->xe,
423 		  REG_FIELD_GET(OAG_OACONTROL_OA_CCS_SELECT_MASK, val) == stream->hwe->instance);
424 	return val;
425 }
426 
427 static void xe_oa_enable(struct xe_oa_stream *stream)
428 {
429 	const struct xe_oa_format *format = stream->oa_buffer.format;
430 	const struct xe_oa_regs *regs;
431 	u32 val;
432 
433 	/*
434 	 * BSpec: 46822: Bit 0. Even if stream->sample is 0, for OAR to function, the OA
435 	 * buffer must be correctly initialized
436 	 */
437 	xe_oa_init_oa_buffer(stream);
438 
439 	regs = __oa_regs(stream);
440 	val = __format_to_oactrl(format, regs->oa_ctrl_counter_select_mask) |
441 		__oa_ccs_select(stream) | OAG_OACONTROL_OA_COUNTER_ENABLE;
442 
443 	if (GRAPHICS_VER(stream->oa->xe) >= 20 &&
444 	    stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG)
445 		val |= OAG_OACONTROL_OA_PES_DISAG_EN;
446 
447 	xe_mmio_write32(stream->gt, regs->oa_ctrl, val);
448 }
449 
450 static void xe_oa_disable(struct xe_oa_stream *stream)
451 {
452 	xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_ctrl, 0);
453 	if (xe_mmio_wait32(stream->gt, __oa_regs(stream)->oa_ctrl,
454 			   OAG_OACONTROL_OA_COUNTER_ENABLE, 0, 50000, NULL, false))
455 		drm_err(&stream->oa->xe->drm,
456 			"wait for OA to be disabled timed out\n");
457 
458 	if (GRAPHICS_VERx100(stream->oa->xe) <= 1270 && GRAPHICS_VERx100(stream->oa->xe) != 1260) {
459 		/* <= XE_METEORLAKE except XE_PVC */
460 		xe_mmio_write32(stream->gt, OA_TLB_INV_CR, 1);
461 		if (xe_mmio_wait32(stream->gt, OA_TLB_INV_CR, 1, 0, 50000, NULL, false))
462 			drm_err(&stream->oa->xe->drm,
463 				"wait for OA tlb invalidate timed out\n");
464 	}
465 }
466 
467 static int xe_oa_wait_unlocked(struct xe_oa_stream *stream)
468 {
469 	/* We might wait indefinitely if periodic sampling is not enabled */
470 	if (!stream->periodic)
471 		return -EINVAL;
472 
473 	return wait_event_interruptible(stream->poll_wq,
474 					xe_oa_buffer_check_unlocked(stream));
475 }
476 
477 #define OASTATUS_RELEVANT_BITS (OASTATUS_MMIO_TRG_Q_FULL | OASTATUS_COUNTER_OVERFLOW | \
478 				OASTATUS_BUFFER_OVERFLOW | OASTATUS_REPORT_LOST)
479 
480 static int __xe_oa_read(struct xe_oa_stream *stream, char __user *buf,
481 			size_t count, size_t *offset)
482 {
483 	/* Only clear our bits to avoid side-effects */
484 	stream->oa_status = xe_mmio_rmw32(stream->gt, __oa_regs(stream)->oa_status,
485 					  OASTATUS_RELEVANT_BITS, 0);
486 	/*
487 	 * Signal to userspace that there is non-zero OA status to read via
488 	 * @DRM_XE_OBSERVATION_IOCTL_STATUS observation stream fd ioctl
489 	 */
490 	if (stream->oa_status & OASTATUS_RELEVANT_BITS)
491 		return -EIO;
492 
493 	return xe_oa_append_reports(stream, buf, count, offset);
494 }
495 
496 static ssize_t xe_oa_read(struct file *file, char __user *buf,
497 			  size_t count, loff_t *ppos)
498 {
499 	struct xe_oa_stream *stream = file->private_data;
500 	size_t offset = 0;
501 	int ret;
502 
503 	/* Can't read from disabled streams */
504 	if (!stream->enabled || !stream->sample)
505 		return -EINVAL;
506 
507 	if (!(file->f_flags & O_NONBLOCK)) {
508 		do {
509 			ret = xe_oa_wait_unlocked(stream);
510 			if (ret)
511 				return ret;
512 
513 			mutex_lock(&stream->stream_lock);
514 			ret = __xe_oa_read(stream, buf, count, &offset);
515 			mutex_unlock(&stream->stream_lock);
516 		} while (!offset && !ret);
517 	} else {
518 		mutex_lock(&stream->stream_lock);
519 		ret = __xe_oa_read(stream, buf, count, &offset);
520 		mutex_unlock(&stream->stream_lock);
521 	}
522 
523 	/*
524 	 * Typically we clear pollin here in order to wait for the new hrtimer callback
525 	 * before unblocking. The exception to this is if __xe_oa_read returns -ENOSPC,
526 	 * which means that more OA data is available than could fit in the user provided
527 	 * buffer. In this case we want the next poll() call to not block.
528 	 *
529 	 * Also in case of -EIO, we have already waited for data before returning
530 	 * -EIO, so need to wait again
531 	 */
532 	if (ret != -ENOSPC && ret != -EIO)
533 		stream->pollin = false;
534 
535 	/* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, -EINVAL, ... */
536 	return offset ?: (ret ?: -EAGAIN);
537 }
538 
539 static __poll_t xe_oa_poll_locked(struct xe_oa_stream *stream,
540 				  struct file *file, poll_table *wait)
541 {
542 	__poll_t events = 0;
543 
544 	poll_wait(file, &stream->poll_wq, wait);
545 
546 	/*
547 	 * We don't explicitly check whether there's something to read here since this
548 	 * path may be hot depending on what else userspace is polling, or on the timeout
549 	 * in use. We rely on hrtimer xe_oa_poll_check_timer_cb to notify us when there
550 	 * are samples to read
551 	 */
552 	if (stream->pollin)
553 		events |= EPOLLIN;
554 
555 	return events;
556 }
557 
558 static __poll_t xe_oa_poll(struct file *file, poll_table *wait)
559 {
560 	struct xe_oa_stream *stream = file->private_data;
561 	__poll_t ret;
562 
563 	mutex_lock(&stream->stream_lock);
564 	ret = xe_oa_poll_locked(stream, file, wait);
565 	mutex_unlock(&stream->stream_lock);
566 
567 	return ret;
568 }
569 
570 static int xe_oa_submit_bb(struct xe_oa_stream *stream, struct xe_bb *bb)
571 {
572 	struct xe_sched_job *job;
573 	struct dma_fence *fence;
574 	long timeout;
575 	int err = 0;
576 
577 	/* Kernel configuration is issued on stream->k_exec_q, not stream->exec_q */
578 	job = xe_bb_create_job(stream->k_exec_q, bb);
579 	if (IS_ERR(job)) {
580 		err = PTR_ERR(job);
581 		goto exit;
582 	}
583 
584 	xe_sched_job_arm(job);
585 	fence = dma_fence_get(&job->drm.s_fence->finished);
586 	xe_sched_job_push(job);
587 
588 	timeout = dma_fence_wait_timeout(fence, false, HZ);
589 	dma_fence_put(fence);
590 	if (timeout < 0)
591 		err = timeout;
592 	else if (!timeout)
593 		err = -ETIME;
594 exit:
595 	return err;
596 }
597 
598 static void write_cs_mi_lri(struct xe_bb *bb, const struct xe_oa_reg *reg_data, u32 n_regs)
599 {
600 	u32 i;
601 
602 #define MI_LOAD_REGISTER_IMM_MAX_REGS (126)
603 
604 	for (i = 0; i < n_regs; i++) {
605 		if ((i % MI_LOAD_REGISTER_IMM_MAX_REGS) == 0) {
606 			u32 n_lri = min_t(u32, n_regs - i,
607 					  MI_LOAD_REGISTER_IMM_MAX_REGS);
608 
609 			bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(n_lri);
610 		}
611 		bb->cs[bb->len++] = reg_data[i].addr.addr;
612 		bb->cs[bb->len++] = reg_data[i].value;
613 	}
614 }
615 
616 static int num_lri_dwords(int num_regs)
617 {
618 	int count = 0;
619 
620 	if (num_regs > 0) {
621 		count += DIV_ROUND_UP(num_regs, MI_LOAD_REGISTER_IMM_MAX_REGS);
622 		count += num_regs * 2;
623 	}
624 
625 	return count;
626 }
627 
628 static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream)
629 {
630 	xe_bo_unpin_map_no_vm(stream->oa_buffer.bo);
631 }
632 
633 static void xe_oa_free_configs(struct xe_oa_stream *stream)
634 {
635 	struct xe_oa_config_bo *oa_bo, *tmp;
636 
637 	xe_oa_config_put(stream->oa_config);
638 	llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)
639 		free_oa_config_bo(oa_bo);
640 }
641 
642 static void xe_oa_store_flex(struct xe_oa_stream *stream, struct xe_lrc *lrc,
643 			     struct xe_bb *bb, const struct flex *flex, u32 count)
644 {
645 	u32 offset = xe_bo_ggtt_addr(lrc->bo);
646 
647 	do {
648 		bb->cs[bb->len++] = MI_STORE_DATA_IMM | BIT(22) /* GGTT */ | 2;
649 		bb->cs[bb->len++] = offset + flex->offset * sizeof(u32);
650 		bb->cs[bb->len++] = 0;
651 		bb->cs[bb->len++] = flex->value;
652 
653 	} while (flex++, --count);
654 }
655 
656 static int xe_oa_modify_ctx_image(struct xe_oa_stream *stream, struct xe_lrc *lrc,
657 				  const struct flex *flex, u32 count)
658 {
659 	struct xe_bb *bb;
660 	int err;
661 
662 	bb = xe_bb_new(stream->gt, 4 * count, false);
663 	if (IS_ERR(bb)) {
664 		err = PTR_ERR(bb);
665 		goto exit;
666 	}
667 
668 	xe_oa_store_flex(stream, lrc, bb, flex, count);
669 
670 	err = xe_oa_submit_bb(stream, bb);
671 	xe_bb_free(bb, NULL);
672 exit:
673 	return err;
674 }
675 
676 static int xe_oa_load_with_lri(struct xe_oa_stream *stream, struct xe_oa_reg *reg_lri)
677 {
678 	struct xe_bb *bb;
679 	int err;
680 
681 	bb = xe_bb_new(stream->gt, 3, false);
682 	if (IS_ERR(bb)) {
683 		err = PTR_ERR(bb);
684 		goto exit;
685 	}
686 
687 	write_cs_mi_lri(bb, reg_lri, 1);
688 
689 	err = xe_oa_submit_bb(stream, bb);
690 	xe_bb_free(bb, NULL);
691 exit:
692 	return err;
693 }
694 
695 static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable)
696 {
697 	const struct xe_oa_format *format = stream->oa_buffer.format;
698 	struct xe_lrc *lrc = stream->exec_q->lrc[0];
699 	u32 regs_offset = xe_lrc_regs_offset(lrc) / sizeof(u32);
700 	u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) |
701 		(enable ? OAR_OACONTROL_COUNTER_ENABLE : 0);
702 
703 	struct flex regs_context[] = {
704 		{
705 			OACTXCONTROL(stream->hwe->mmio_base),
706 			stream->oa->ctx_oactxctrl_offset[stream->hwe->class] + 1,
707 			enable ? OA_COUNTER_RESUME : 0,
708 		},
709 		{
710 			RING_CONTEXT_CONTROL(stream->hwe->mmio_base),
711 			regs_offset + CTX_CONTEXT_CONTROL,
712 			_MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE,
713 				      enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0)
714 		},
715 	};
716 	struct xe_oa_reg reg_lri = { OAR_OACONTROL, oacontrol };
717 	int err;
718 
719 	/* Modify stream hwe context image with regs_context */
720 	err = xe_oa_modify_ctx_image(stream, stream->exec_q->lrc[0],
721 				     regs_context, ARRAY_SIZE(regs_context));
722 	if (err)
723 		return err;
724 
725 	/* Apply reg_lri using LRI */
726 	return xe_oa_load_with_lri(stream, &reg_lri);
727 }
728 
729 static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable)
730 {
731 	const struct xe_oa_format *format = stream->oa_buffer.format;
732 	struct xe_lrc *lrc = stream->exec_q->lrc[0];
733 	u32 regs_offset = xe_lrc_regs_offset(lrc) / sizeof(u32);
734 	u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) |
735 		(enable ? OAR_OACONTROL_COUNTER_ENABLE : 0);
736 	struct flex regs_context[] = {
737 		{
738 			OACTXCONTROL(stream->hwe->mmio_base),
739 			stream->oa->ctx_oactxctrl_offset[stream->hwe->class] + 1,
740 			enable ? OA_COUNTER_RESUME : 0,
741 		},
742 		{
743 			RING_CONTEXT_CONTROL(stream->hwe->mmio_base),
744 			regs_offset + CTX_CONTEXT_CONTROL,
745 			_MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE,
746 				      enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0) |
747 			_MASKED_FIELD(CTX_CTRL_RUN_ALONE,
748 				      enable ? CTX_CTRL_RUN_ALONE : 0),
749 		},
750 	};
751 	struct xe_oa_reg reg_lri = { OAC_OACONTROL, oacontrol };
752 	int err;
753 
754 	/* Set ccs select to enable programming of OAC_OACONTROL */
755 	xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_ctrl, __oa_ccs_select(stream));
756 
757 	/* Modify stream hwe context image with regs_context */
758 	err = xe_oa_modify_ctx_image(stream, stream->exec_q->lrc[0],
759 				     regs_context, ARRAY_SIZE(regs_context));
760 	if (err)
761 		return err;
762 
763 	/* Apply reg_lri using LRI */
764 	return xe_oa_load_with_lri(stream, &reg_lri);
765 }
766 
767 static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable)
768 {
769 	switch (stream->hwe->class) {
770 	case XE_ENGINE_CLASS_RENDER:
771 		return xe_oa_configure_oar_context(stream, enable);
772 	case XE_ENGINE_CLASS_COMPUTE:
773 		return xe_oa_configure_oac_context(stream, enable);
774 	default:
775 		/* Video engines do not support MI_REPORT_PERF_COUNT */
776 		return 0;
777 	}
778 }
779 
780 #define HAS_OA_BPC_REPORTING(xe) (GRAPHICS_VERx100(xe) >= 1255)
781 
782 static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable)
783 {
784 	return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_MMIO_TRG,
785 			     enable && stream && stream->sample ?
786 			     0 : OAG_OA_DEBUG_DISABLE_MMIO_TRG);
787 }
788 
789 static void xe_oa_disable_metric_set(struct xe_oa_stream *stream)
790 {
791 	u32 sqcnt1;
792 
793 	/*
794 	 * Wa_1508761755:xehpsdv, dg2
795 	 * Enable thread stall DOP gating and EU DOP gating.
796 	 */
797 	if (stream->oa->xe->info.platform == XE_DG2) {
798 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN,
799 					  _MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE));
800 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2,
801 					  _MASKED_BIT_DISABLE(DISABLE_DOP_GATING));
802 	}
803 
804 	xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_debug,
805 			oag_configure_mmio_trigger(stream, false));
806 
807 	/* disable the context save/restore or OAR counters */
808 	if (stream->exec_q)
809 		xe_oa_configure_oa_context(stream, false);
810 
811 	/* Make sure we disable noa to save power. */
812 	xe_mmio_rmw32(stream->gt, RPM_CONFIG1, GT_NOA_ENABLE, 0);
813 
814 	sqcnt1 = SQCNT1_PMON_ENABLE |
815 		 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0);
816 
817 	/* Reset PMON Enable to save power. */
818 	xe_mmio_rmw32(stream->gt, XELPMP_SQCNT1, sqcnt1, 0);
819 }
820 
821 static void xe_oa_stream_destroy(struct xe_oa_stream *stream)
822 {
823 	struct xe_oa_unit *u = stream->hwe->oa_unit;
824 	struct xe_gt *gt = stream->hwe->gt;
825 
826 	if (WARN_ON(stream != u->exclusive_stream))
827 		return;
828 
829 	WRITE_ONCE(u->exclusive_stream, NULL);
830 
831 	mutex_destroy(&stream->stream_lock);
832 
833 	xe_oa_disable_metric_set(stream);
834 	xe_exec_queue_put(stream->k_exec_q);
835 
836 	xe_oa_free_oa_buffer(stream);
837 
838 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
839 	xe_pm_runtime_put(stream->oa->xe);
840 
841 	/* Wa_1509372804:pvc: Unset the override of GUCRC mode to enable rc6 */
842 	if (stream->override_gucrc)
843 		xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(&gt->uc.guc.pc));
844 
845 	xe_oa_free_configs(stream);
846 }
847 
848 static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream)
849 {
850 	struct xe_bo *bo;
851 
852 	BUILD_BUG_ON_NOT_POWER_OF_2(XE_OA_BUFFER_SIZE);
853 	BUILD_BUG_ON(XE_OA_BUFFER_SIZE < SZ_128K || XE_OA_BUFFER_SIZE > SZ_16M);
854 
855 	bo = xe_bo_create_pin_map(stream->oa->xe, stream->gt->tile, NULL,
856 				  XE_OA_BUFFER_SIZE, ttm_bo_type_kernel,
857 				  XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT);
858 	if (IS_ERR(bo))
859 		return PTR_ERR(bo);
860 
861 	stream->oa_buffer.bo = bo;
862 	/* mmap implementation requires OA buffer to be in system memory */
863 	xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0);
864 	stream->oa_buffer.vaddr = bo->vmap.vaddr;
865 	return 0;
866 }
867 
868 static struct xe_oa_config_bo *
869 __xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config)
870 {
871 	struct xe_oa_config_bo *oa_bo;
872 	size_t config_length;
873 	struct xe_bb *bb;
874 
875 	oa_bo = kzalloc(sizeof(*oa_bo), GFP_KERNEL);
876 	if (!oa_bo)
877 		return ERR_PTR(-ENOMEM);
878 
879 	config_length = num_lri_dwords(oa_config->regs_len);
880 	config_length = ALIGN(sizeof(u32) * config_length, XE_PAGE_SIZE) / sizeof(u32);
881 
882 	bb = xe_bb_new(stream->gt, config_length, false);
883 	if (IS_ERR(bb))
884 		goto err_free;
885 
886 	write_cs_mi_lri(bb, oa_config->regs, oa_config->regs_len);
887 
888 	oa_bo->bb = bb;
889 	oa_bo->oa_config = xe_oa_config_get(oa_config);
890 	llist_add(&oa_bo->node, &stream->oa_config_bos);
891 
892 	return oa_bo;
893 err_free:
894 	kfree(oa_bo);
895 	return ERR_CAST(bb);
896 }
897 
898 static struct xe_oa_config_bo *
899 xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config)
900 {
901 	struct xe_oa_config_bo *oa_bo;
902 
903 	/* Look for the buffer in the already allocated BOs attached to the stream */
904 	llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {
905 		if (oa_bo->oa_config == oa_config &&
906 		    memcmp(oa_bo->oa_config->uuid, oa_config->uuid,
907 			   sizeof(oa_config->uuid)) == 0)
908 			goto out;
909 	}
910 
911 	oa_bo = __xe_oa_alloc_config_buffer(stream, oa_config);
912 out:
913 	return oa_bo;
914 }
915 
916 static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config *config)
917 {
918 #define NOA_PROGRAM_ADDITIONAL_DELAY_US 500
919 	struct xe_oa_config_bo *oa_bo;
920 	int err, us = NOA_PROGRAM_ADDITIONAL_DELAY_US;
921 
922 	oa_bo = xe_oa_alloc_config_buffer(stream, config);
923 	if (IS_ERR(oa_bo)) {
924 		err = PTR_ERR(oa_bo);
925 		goto exit;
926 	}
927 
928 	err = xe_oa_submit_bb(stream, oa_bo->bb);
929 
930 	/* Additional empirical delay needed for NOA programming after registers are written */
931 	usleep_range(us, 2 * us);
932 exit:
933 	return err;
934 }
935 
936 static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream)
937 {
938 	/* If user didn't require OA reports, ask HW not to emit ctx switch reports */
939 	return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS,
940 			     stream->sample ?
941 			     0 : OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
942 }
943 
944 static int xe_oa_enable_metric_set(struct xe_oa_stream *stream)
945 {
946 	u32 oa_debug, sqcnt1;
947 	int ret;
948 
949 	/*
950 	 * Wa_1508761755:xehpsdv, dg2
951 	 * EU NOA signals behave incorrectly if EU clock gating is enabled.
952 	 * Disable thread stall DOP gating and EU DOP gating.
953 	 */
954 	if (stream->oa->xe->info.platform == XE_DG2) {
955 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN,
956 					  _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
957 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2,
958 					  _MASKED_BIT_ENABLE(DISABLE_DOP_GATING));
959 	}
960 
961 	/* Disable clk ratio reports */
962 	oa_debug = OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
963 		OAG_OA_DEBUG_INCLUDE_CLK_RATIO;
964 
965 	if (GRAPHICS_VER(stream->oa->xe) >= 20)
966 		oa_debug |=
967 			/* The three bits below are needed to get PEC counters running */
968 			OAG_OA_DEBUG_START_TRIGGER_SCOPE_CONTROL |
969 			OAG_OA_DEBUG_DISABLE_START_TRG_2_COUNT_QUAL |
970 			OAG_OA_DEBUG_DISABLE_START_TRG_1_COUNT_QUAL;
971 
972 	xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_debug,
973 			_MASKED_BIT_ENABLE(oa_debug) |
974 			oag_report_ctx_switches(stream) |
975 			oag_configure_mmio_trigger(stream, true));
976 
977 	xe_mmio_write32(stream->gt, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ?
978 			(OAG_OAGLBCTXCTRL_COUNTER_RESUME |
979 			 OAG_OAGLBCTXCTRL_TIMER_ENABLE |
980 			 REG_FIELD_PREP(OAG_OAGLBCTXCTRL_TIMER_PERIOD_MASK,
981 					stream->period_exponent)) : 0);
982 
983 	/*
984 	 * Initialize Super Queue Internal Cnt Register
985 	 * Set PMON Enable in order to collect valid metrics
986 	 * Enable bytes per clock reporting
987 	 */
988 	sqcnt1 = SQCNT1_PMON_ENABLE |
989 		 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0);
990 
991 	xe_mmio_rmw32(stream->gt, XELPMP_SQCNT1, 0, sqcnt1);
992 
993 	/* Configure OAR/OAC */
994 	if (stream->exec_q) {
995 		ret = xe_oa_configure_oa_context(stream, true);
996 		if (ret)
997 			return ret;
998 	}
999 
1000 	return xe_oa_emit_oa_config(stream, stream->oa_config);
1001 }
1002 
1003 static void xe_oa_stream_enable(struct xe_oa_stream *stream)
1004 {
1005 	stream->pollin = false;
1006 
1007 	xe_oa_enable(stream);
1008 
1009 	if (stream->sample)
1010 		hrtimer_start(&stream->poll_check_timer,
1011 			      ns_to_ktime(stream->poll_period_ns),
1012 			      HRTIMER_MODE_REL_PINNED);
1013 }
1014 
1015 static void xe_oa_stream_disable(struct xe_oa_stream *stream)
1016 {
1017 	xe_oa_disable(stream);
1018 
1019 	if (stream->sample)
1020 		hrtimer_cancel(&stream->poll_check_timer);
1021 }
1022 
1023 static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream)
1024 {
1025 	struct xe_exec_queue *q = stream->exec_q;
1026 	int ret1, ret2;
1027 
1028 	/* Best effort recovery: try to revert both to original, irrespective of error */
1029 	ret1 = q->ops->set_timeslice(q, stream->hwe->eclass->sched_props.timeslice_us);
1030 	ret2 = q->ops->set_preempt_timeout(q, stream->hwe->eclass->sched_props.preempt_timeout_us);
1031 	if (ret1 || ret2)
1032 		goto err;
1033 	return 0;
1034 err:
1035 	drm_dbg(&stream->oa->xe->drm, "%s failed ret1 %d ret2 %d\n", __func__, ret1, ret2);
1036 	return ret1 ?: ret2;
1037 }
1038 
1039 static int xe_oa_disable_preempt_timeslice(struct xe_oa_stream *stream)
1040 {
1041 	struct xe_exec_queue *q = stream->exec_q;
1042 	int ret;
1043 
1044 	/* Setting values to 0 will disable timeslice and preempt_timeout */
1045 	ret = q->ops->set_timeslice(q, 0);
1046 	if (ret)
1047 		goto err;
1048 
1049 	ret = q->ops->set_preempt_timeout(q, 0);
1050 	if (ret)
1051 		goto err;
1052 
1053 	return 0;
1054 err:
1055 	xe_oa_enable_preempt_timeslice(stream);
1056 	drm_dbg(&stream->oa->xe->drm, "%s failed %d\n", __func__, ret);
1057 	return ret;
1058 }
1059 
1060 static int xe_oa_enable_locked(struct xe_oa_stream *stream)
1061 {
1062 	if (stream->enabled)
1063 		return 0;
1064 
1065 	if (stream->no_preempt) {
1066 		int ret = xe_oa_disable_preempt_timeslice(stream);
1067 
1068 		if (ret)
1069 			return ret;
1070 	}
1071 
1072 	xe_oa_stream_enable(stream);
1073 
1074 	stream->enabled = true;
1075 	return 0;
1076 }
1077 
1078 static int xe_oa_disable_locked(struct xe_oa_stream *stream)
1079 {
1080 	int ret = 0;
1081 
1082 	if (!stream->enabled)
1083 		return 0;
1084 
1085 	xe_oa_stream_disable(stream);
1086 
1087 	if (stream->no_preempt)
1088 		ret = xe_oa_enable_preempt_timeslice(stream);
1089 
1090 	stream->enabled = false;
1091 	return ret;
1092 }
1093 
1094 static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg)
1095 {
1096 	struct drm_xe_ext_set_property ext;
1097 	long ret = stream->oa_config->id;
1098 	struct xe_oa_config *config;
1099 	int err;
1100 
1101 	err = __copy_from_user(&ext, u64_to_user_ptr(arg), sizeof(ext));
1102 	if (XE_IOCTL_DBG(stream->oa->xe, err))
1103 		return -EFAULT;
1104 
1105 	if (XE_IOCTL_DBG(stream->oa->xe, ext.pad) ||
1106 	    XE_IOCTL_DBG(stream->oa->xe, ext.base.name != DRM_XE_OA_EXTENSION_SET_PROPERTY) ||
1107 	    XE_IOCTL_DBG(stream->oa->xe, ext.base.next_extension) ||
1108 	    XE_IOCTL_DBG(stream->oa->xe, ext.property != DRM_XE_OA_PROPERTY_OA_METRIC_SET))
1109 		return -EINVAL;
1110 
1111 	config = xe_oa_get_oa_config(stream->oa, ext.value);
1112 	if (!config)
1113 		return -ENODEV;
1114 
1115 	if (config != stream->oa_config) {
1116 		err = xe_oa_emit_oa_config(stream, config);
1117 		if (!err)
1118 			config = xchg(&stream->oa_config, config);
1119 		else
1120 			ret = err;
1121 	}
1122 
1123 	xe_oa_config_put(config);
1124 
1125 	return ret;
1126 }
1127 
1128 static long xe_oa_status_locked(struct xe_oa_stream *stream, unsigned long arg)
1129 {
1130 	struct drm_xe_oa_stream_status status = {};
1131 	void __user *uaddr = (void __user *)arg;
1132 
1133 	/* Map from register to uapi bits */
1134 	if (stream->oa_status & OASTATUS_REPORT_LOST)
1135 		status.oa_status |= DRM_XE_OASTATUS_REPORT_LOST;
1136 	if (stream->oa_status & OASTATUS_BUFFER_OVERFLOW)
1137 		status.oa_status |= DRM_XE_OASTATUS_BUFFER_OVERFLOW;
1138 	if (stream->oa_status & OASTATUS_COUNTER_OVERFLOW)
1139 		status.oa_status |= DRM_XE_OASTATUS_COUNTER_OVERFLOW;
1140 	if (stream->oa_status & OASTATUS_MMIO_TRG_Q_FULL)
1141 		status.oa_status |= DRM_XE_OASTATUS_MMIO_TRG_Q_FULL;
1142 
1143 	if (copy_to_user(uaddr, &status, sizeof(status)))
1144 		return -EFAULT;
1145 
1146 	return 0;
1147 }
1148 
1149 static long xe_oa_info_locked(struct xe_oa_stream *stream, unsigned long arg)
1150 {
1151 	struct drm_xe_oa_stream_info info = { .oa_buf_size = XE_OA_BUFFER_SIZE, };
1152 	void __user *uaddr = (void __user *)arg;
1153 
1154 	if (copy_to_user(uaddr, &info, sizeof(info)))
1155 		return -EFAULT;
1156 
1157 	return 0;
1158 }
1159 
1160 static long xe_oa_ioctl_locked(struct xe_oa_stream *stream,
1161 			       unsigned int cmd,
1162 			       unsigned long arg)
1163 {
1164 	switch (cmd) {
1165 	case DRM_XE_OBSERVATION_IOCTL_ENABLE:
1166 		return xe_oa_enable_locked(stream);
1167 	case DRM_XE_OBSERVATION_IOCTL_DISABLE:
1168 		return xe_oa_disable_locked(stream);
1169 	case DRM_XE_OBSERVATION_IOCTL_CONFIG:
1170 		return xe_oa_config_locked(stream, arg);
1171 	case DRM_XE_OBSERVATION_IOCTL_STATUS:
1172 		return xe_oa_status_locked(stream, arg);
1173 	case DRM_XE_OBSERVATION_IOCTL_INFO:
1174 		return xe_oa_info_locked(stream, arg);
1175 	}
1176 
1177 	return -EINVAL;
1178 }
1179 
1180 static long xe_oa_ioctl(struct file *file,
1181 			unsigned int cmd,
1182 			unsigned long arg)
1183 {
1184 	struct xe_oa_stream *stream = file->private_data;
1185 	long ret;
1186 
1187 	mutex_lock(&stream->stream_lock);
1188 	ret = xe_oa_ioctl_locked(stream, cmd, arg);
1189 	mutex_unlock(&stream->stream_lock);
1190 
1191 	return ret;
1192 }
1193 
1194 static void xe_oa_destroy_locked(struct xe_oa_stream *stream)
1195 {
1196 	if (stream->enabled)
1197 		xe_oa_disable_locked(stream);
1198 
1199 	xe_oa_stream_destroy(stream);
1200 
1201 	if (stream->exec_q)
1202 		xe_exec_queue_put(stream->exec_q);
1203 
1204 	kfree(stream);
1205 }
1206 
1207 static int xe_oa_release(struct inode *inode, struct file *file)
1208 {
1209 	struct xe_oa_stream *stream = file->private_data;
1210 	struct xe_gt *gt = stream->gt;
1211 
1212 	mutex_lock(&gt->oa.gt_lock);
1213 	xe_oa_destroy_locked(stream);
1214 	mutex_unlock(&gt->oa.gt_lock);
1215 
1216 	/* Release the reference the OA stream kept on the driver */
1217 	drm_dev_put(&gt_to_xe(gt)->drm);
1218 
1219 	return 0;
1220 }
1221 
1222 static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma)
1223 {
1224 	struct xe_oa_stream *stream = file->private_data;
1225 	struct xe_bo *bo = stream->oa_buffer.bo;
1226 	unsigned long start = vma->vm_start;
1227 	int i, ret;
1228 
1229 	if (xe_observation_paranoid && !perfmon_capable()) {
1230 		drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n");
1231 		return -EACCES;
1232 	}
1233 
1234 	/* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */
1235 	if (vma->vm_end - vma->vm_start != XE_OA_BUFFER_SIZE) {
1236 		drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n");
1237 		return -EINVAL;
1238 	}
1239 
1240 	/*
1241 	 * Only support VM_READ, enforce MAP_PRIVATE by checking for
1242 	 * VM_MAYSHARE, don't copy the vma on fork
1243 	 */
1244 	if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_SHARED | VM_MAYSHARE)) {
1245 		drm_dbg(&stream->oa->xe->drm, "mmap must be read only\n");
1246 		return -EINVAL;
1247 	}
1248 	vm_flags_mod(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY,
1249 		     VM_MAYWRITE | VM_MAYEXEC);
1250 
1251 	xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages ==
1252 		  (vma->vm_end - vma->vm_start) >> PAGE_SHIFT);
1253 	for (i = 0; i < bo->ttm.ttm->num_pages; i++) {
1254 		ret = remap_pfn_range(vma, start, page_to_pfn(bo->ttm.ttm->pages[i]),
1255 				      PAGE_SIZE, vma->vm_page_prot);
1256 		if (ret)
1257 			break;
1258 
1259 		start += PAGE_SIZE;
1260 	}
1261 
1262 	return ret;
1263 }
1264 
1265 static const struct file_operations xe_oa_fops = {
1266 	.owner		= THIS_MODULE,
1267 	.llseek		= no_llseek,
1268 	.release	= xe_oa_release,
1269 	.poll		= xe_oa_poll,
1270 	.read		= xe_oa_read,
1271 	.unlocked_ioctl	= xe_oa_ioctl,
1272 	.mmap		= xe_oa_mmap,
1273 };
1274 
1275 static bool engine_supports_mi_query(struct xe_hw_engine *hwe)
1276 {
1277 	return hwe->class == XE_ENGINE_CLASS_RENDER ||
1278 		hwe->class == XE_ENGINE_CLASS_COMPUTE;
1279 }
1280 
1281 static bool xe_oa_find_reg_in_lri(u32 *state, u32 reg, u32 *offset, u32 end)
1282 {
1283 	u32 idx = *offset;
1284 	u32 len = min(MI_LRI_LEN(state[idx]) + idx, end);
1285 	bool found = false;
1286 
1287 	idx++;
1288 	for (; idx < len; idx += 2) {
1289 		if (state[idx] == reg) {
1290 			found = true;
1291 			break;
1292 		}
1293 	}
1294 
1295 	*offset = idx;
1296 	return found;
1297 }
1298 
1299 #define IS_MI_LRI_CMD(x) (REG_FIELD_GET(MI_OPCODE, (x)) == \
1300 			  REG_FIELD_GET(MI_OPCODE, MI_LOAD_REGISTER_IMM))
1301 
1302 static u32 xe_oa_context_image_offset(struct xe_oa_stream *stream, u32 reg)
1303 {
1304 	struct xe_lrc *lrc = stream->exec_q->lrc[0];
1305 	u32 len = (xe_gt_lrc_size(stream->gt, stream->hwe->class) +
1306 		   lrc->ring.size) / sizeof(u32);
1307 	u32 offset = xe_lrc_regs_offset(lrc) / sizeof(u32);
1308 	u32 *state = (u32 *)lrc->bo->vmap.vaddr;
1309 
1310 	if (drm_WARN_ON(&stream->oa->xe->drm, !state))
1311 		return U32_MAX;
1312 
1313 	for (; offset < len; ) {
1314 		if (IS_MI_LRI_CMD(state[offset])) {
1315 			/*
1316 			 * We expect reg-value pairs in MI_LRI command, so
1317 			 * MI_LRI_LEN() should be even
1318 			 */
1319 			drm_WARN_ON(&stream->oa->xe->drm,
1320 				    MI_LRI_LEN(state[offset]) & 0x1);
1321 
1322 			if (xe_oa_find_reg_in_lri(state, reg, &offset, len))
1323 				break;
1324 		} else {
1325 			offset++;
1326 		}
1327 	}
1328 
1329 	return offset < len ? offset : U32_MAX;
1330 }
1331 
1332 static int xe_oa_set_ctx_ctrl_offset(struct xe_oa_stream *stream)
1333 {
1334 	struct xe_reg reg = OACTXCONTROL(stream->hwe->mmio_base);
1335 	u32 offset = stream->oa->ctx_oactxctrl_offset[stream->hwe->class];
1336 
1337 	/* Do this only once. Failure is stored as offset of U32_MAX */
1338 	if (offset)
1339 		goto exit;
1340 
1341 	offset = xe_oa_context_image_offset(stream, reg.addr);
1342 	stream->oa->ctx_oactxctrl_offset[stream->hwe->class] = offset;
1343 
1344 	drm_dbg(&stream->oa->xe->drm, "%s oa ctx control at 0x%08x dword offset\n",
1345 		stream->hwe->name, offset);
1346 exit:
1347 	return offset && offset != U32_MAX ? 0 : -ENODEV;
1348 }
1349 
1350 static int xe_oa_stream_init(struct xe_oa_stream *stream,
1351 			     struct xe_oa_open_param *param)
1352 {
1353 	struct xe_oa_unit *u = param->hwe->oa_unit;
1354 	struct xe_gt *gt = param->hwe->gt;
1355 	int ret;
1356 
1357 	stream->exec_q = param->exec_q;
1358 	stream->poll_period_ns = DEFAULT_POLL_PERIOD_NS;
1359 	stream->hwe = param->hwe;
1360 	stream->gt = stream->hwe->gt;
1361 	stream->oa_buffer.format = &stream->oa->oa_formats[param->oa_format];
1362 
1363 	stream->sample = param->sample;
1364 	stream->periodic = param->period_exponent > 0;
1365 	stream->period_exponent = param->period_exponent;
1366 	stream->no_preempt = param->no_preempt;
1367 
1368 	/*
1369 	 * For Xe2+, when overrun mode is enabled, there are no partial reports at the end
1370 	 * of buffer, making the OA buffer effectively a non-power-of-2 size circular
1371 	 * buffer whose size, circ_size, is a multiple of the report size
1372 	 */
1373 	if (GRAPHICS_VER(stream->oa->xe) >= 20 &&
1374 	    stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG && stream->sample)
1375 		stream->oa_buffer.circ_size =
1376 			XE_OA_BUFFER_SIZE - XE_OA_BUFFER_SIZE % stream->oa_buffer.format->size;
1377 	else
1378 		stream->oa_buffer.circ_size = XE_OA_BUFFER_SIZE;
1379 
1380 	if (stream->exec_q && engine_supports_mi_query(stream->hwe)) {
1381 		/* If we don't find the context offset, just return error */
1382 		ret = xe_oa_set_ctx_ctrl_offset(stream);
1383 		if (ret) {
1384 			drm_err(&stream->oa->xe->drm,
1385 				"xe_oa_set_ctx_ctrl_offset failed for %s\n",
1386 				stream->hwe->name);
1387 			goto exit;
1388 		}
1389 	}
1390 
1391 	stream->oa_config = xe_oa_get_oa_config(stream->oa, param->metric_set);
1392 	if (!stream->oa_config) {
1393 		drm_dbg(&stream->oa->xe->drm, "Invalid OA config id=%i\n", param->metric_set);
1394 		ret = -EINVAL;
1395 		goto exit;
1396 	}
1397 
1398 	/*
1399 	 * Wa_1509372804:pvc
1400 	 *
1401 	 * GuC reset of engines causes OA to lose configuration
1402 	 * state. Prevent this by overriding GUCRC mode.
1403 	 */
1404 	if (stream->oa->xe->info.platform == XE_PVC) {
1405 		ret = xe_guc_pc_override_gucrc_mode(&gt->uc.guc.pc,
1406 						    SLPC_GUCRC_MODE_GUCRC_NO_RC6);
1407 		if (ret)
1408 			goto err_free_configs;
1409 
1410 		stream->override_gucrc = true;
1411 	}
1412 
1413 	/* Take runtime pm ref and forcewake to disable RC6 */
1414 	xe_pm_runtime_get(stream->oa->xe);
1415 	XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL));
1416 
1417 	ret = xe_oa_alloc_oa_buffer(stream);
1418 	if (ret)
1419 		goto err_fw_put;
1420 
1421 	stream->k_exec_q = xe_exec_queue_create(stream->oa->xe, NULL,
1422 						BIT(stream->hwe->logical_instance), 1,
1423 						stream->hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
1424 	if (IS_ERR(stream->k_exec_q)) {
1425 		ret = PTR_ERR(stream->k_exec_q);
1426 		drm_err(&stream->oa->xe->drm, "gt%d, hwe %s, xe_exec_queue_create failed=%d",
1427 			stream->gt->info.id, stream->hwe->name, ret);
1428 		goto err_free_oa_buf;
1429 	}
1430 
1431 	ret = xe_oa_enable_metric_set(stream);
1432 	if (ret) {
1433 		drm_dbg(&stream->oa->xe->drm, "Unable to enable metric set\n");
1434 		goto err_put_k_exec_q;
1435 	}
1436 
1437 	drm_dbg(&stream->oa->xe->drm, "opening stream oa config uuid=%s\n",
1438 		stream->oa_config->uuid);
1439 
1440 	WRITE_ONCE(u->exclusive_stream, stream);
1441 
1442 	hrtimer_init(&stream->poll_check_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1443 	stream->poll_check_timer.function = xe_oa_poll_check_timer_cb;
1444 	init_waitqueue_head(&stream->poll_wq);
1445 
1446 	spin_lock_init(&stream->oa_buffer.ptr_lock);
1447 	mutex_init(&stream->stream_lock);
1448 
1449 	return 0;
1450 
1451 err_put_k_exec_q:
1452 	xe_oa_disable_metric_set(stream);
1453 	xe_exec_queue_put(stream->k_exec_q);
1454 err_free_oa_buf:
1455 	xe_oa_free_oa_buffer(stream);
1456 err_fw_put:
1457 	XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL));
1458 	xe_pm_runtime_put(stream->oa->xe);
1459 	if (stream->override_gucrc)
1460 		xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(&gt->uc.guc.pc));
1461 err_free_configs:
1462 	xe_oa_free_configs(stream);
1463 exit:
1464 	return ret;
1465 }
1466 
1467 static int xe_oa_stream_open_ioctl_locked(struct xe_oa *oa,
1468 					  struct xe_oa_open_param *param)
1469 {
1470 	struct xe_oa_stream *stream;
1471 	int stream_fd;
1472 	int ret;
1473 
1474 	/* We currently only allow exclusive access */
1475 	if (param->hwe->oa_unit->exclusive_stream) {
1476 		drm_dbg(&oa->xe->drm, "OA unit already in use\n");
1477 		ret = -EBUSY;
1478 		goto exit;
1479 	}
1480 
1481 	stream = kzalloc(sizeof(*stream), GFP_KERNEL);
1482 	if (!stream) {
1483 		ret = -ENOMEM;
1484 		goto exit;
1485 	}
1486 
1487 	stream->oa = oa;
1488 	ret = xe_oa_stream_init(stream, param);
1489 	if (ret)
1490 		goto err_free;
1491 
1492 	if (!param->disabled) {
1493 		ret = xe_oa_enable_locked(stream);
1494 		if (ret)
1495 			goto err_destroy;
1496 	}
1497 
1498 	stream_fd = anon_inode_getfd("[xe_oa]", &xe_oa_fops, stream, 0);
1499 	if (stream_fd < 0) {
1500 		ret = stream_fd;
1501 		goto err_disable;
1502 	}
1503 
1504 	/* Hold a reference on the drm device till stream_fd is released */
1505 	drm_dev_get(&stream->oa->xe->drm);
1506 
1507 	return stream_fd;
1508 err_disable:
1509 	if (!param->disabled)
1510 		xe_oa_disable_locked(stream);
1511 err_destroy:
1512 	xe_oa_stream_destroy(stream);
1513 err_free:
1514 	kfree(stream);
1515 exit:
1516 	return ret;
1517 }
1518 
1519 /**
1520  * xe_oa_timestamp_frequency - Return OA timestamp frequency
1521  * @gt: @xe_gt
1522  *
1523  * OA timestamp frequency = CS timestamp frequency in most platforms. On some
1524  * platforms OA unit ignores the CTC_SHIFT and the 2 timestamps differ. In such
1525  * cases, return the adjusted CS timestamp frequency to the user.
1526  */
1527 u32 xe_oa_timestamp_frequency(struct xe_gt *gt)
1528 {
1529 	u32 reg, shift;
1530 
1531 	/*
1532 	 * Wa_18013179988:dg2
1533 	 * Wa_14015568240:pvc
1534 	 * Wa_14015846243:mtl
1535 	 */
1536 	switch (gt_to_xe(gt)->info.platform) {
1537 	case XE_DG2:
1538 	case XE_PVC:
1539 	case XE_METEORLAKE:
1540 		xe_pm_runtime_get(gt_to_xe(gt));
1541 		reg = xe_mmio_read32(gt, RPM_CONFIG0);
1542 		xe_pm_runtime_put(gt_to_xe(gt));
1543 
1544 		shift = REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, reg);
1545 		return gt->info.reference_clock << (3 - shift);
1546 
1547 	default:
1548 		return gt->info.reference_clock;
1549 	}
1550 }
1551 
1552 static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent)
1553 {
1554 	u64 nom = (2ULL << exponent) * NSEC_PER_SEC;
1555 	u32 den = xe_oa_timestamp_frequency(gt);
1556 
1557 	return div_u64(nom + den - 1, den);
1558 }
1559 
1560 static bool engine_supports_oa_format(const struct xe_hw_engine *hwe, int type)
1561 {
1562 	switch (hwe->oa_unit->type) {
1563 	case DRM_XE_OA_UNIT_TYPE_OAG:
1564 		return type == DRM_XE_OA_FMT_TYPE_OAG || type == DRM_XE_OA_FMT_TYPE_OAR ||
1565 			type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC;
1566 	case DRM_XE_OA_UNIT_TYPE_OAM:
1567 		return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC;
1568 	default:
1569 		return false;
1570 	}
1571 }
1572 
1573 static int decode_oa_format(struct xe_oa *oa, u64 fmt, enum xe_oa_format_name *name)
1574 {
1575 	u32 counter_size = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SIZE, fmt);
1576 	u32 counter_sel = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SEL, fmt);
1577 	u32 bc_report = FIELD_GET(DRM_XE_OA_FORMAT_MASK_BC_REPORT, fmt);
1578 	u32 type = FIELD_GET(DRM_XE_OA_FORMAT_MASK_FMT_TYPE, fmt);
1579 	int idx;
1580 
1581 	for_each_set_bit(idx, oa->format_mask, __XE_OA_FORMAT_MAX) {
1582 		const struct xe_oa_format *f = &oa->oa_formats[idx];
1583 
1584 		if (counter_size == f->counter_size && bc_report == f->bc_report &&
1585 		    type == f->type && counter_sel == f->counter_select) {
1586 			*name = idx;
1587 			return 0;
1588 		}
1589 	}
1590 
1591 	return -EINVAL;
1592 }
1593 
1594 /**
1595  * xe_oa_unit_id - Return OA unit ID for a hardware engine
1596  * @hwe: @xe_hw_engine
1597  *
1598  * Return OA unit ID for a hardware engine when available
1599  */
1600 u16 xe_oa_unit_id(struct xe_hw_engine *hwe)
1601 {
1602 	return hwe->oa_unit && hwe->oa_unit->num_engines ?
1603 		hwe->oa_unit->oa_unit_id : U16_MAX;
1604 }
1605 
1606 static int xe_oa_assign_hwe(struct xe_oa *oa, struct xe_oa_open_param *param)
1607 {
1608 	struct xe_gt *gt;
1609 	int i, ret = 0;
1610 
1611 	if (param->exec_q) {
1612 		/* When we have an exec_q, get hwe from the exec_q */
1613 		param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class,
1614 					     param->engine_instance, true);
1615 	} else {
1616 		struct xe_hw_engine *hwe;
1617 		enum xe_hw_engine_id id;
1618 
1619 		/* Else just get the first hwe attached to the oa unit */
1620 		for_each_gt(gt, oa->xe, i) {
1621 			for_each_hw_engine(hwe, gt, id) {
1622 				if (xe_oa_unit_id(hwe) == param->oa_unit_id) {
1623 					param->hwe = hwe;
1624 					goto out;
1625 				}
1626 			}
1627 		}
1628 	}
1629 out:
1630 	if (!param->hwe || xe_oa_unit_id(param->hwe) != param->oa_unit_id) {
1631 		drm_dbg(&oa->xe->drm, "Unable to find hwe (%d, %d) for OA unit ID %d\n",
1632 			param->exec_q ? param->exec_q->class : -1,
1633 			param->engine_instance, param->oa_unit_id);
1634 		ret = -EINVAL;
1635 	}
1636 
1637 	return ret;
1638 }
1639 
1640 static int xe_oa_set_prop_oa_unit_id(struct xe_oa *oa, u64 value,
1641 				     struct xe_oa_open_param *param)
1642 {
1643 	if (value >= oa->oa_unit_ids) {
1644 		drm_dbg(&oa->xe->drm, "OA unit ID out of range %lld\n", value);
1645 		return -EINVAL;
1646 	}
1647 	param->oa_unit_id = value;
1648 	return 0;
1649 }
1650 
1651 static int xe_oa_set_prop_sample_oa(struct xe_oa *oa, u64 value,
1652 				    struct xe_oa_open_param *param)
1653 {
1654 	param->sample = value;
1655 	return 0;
1656 }
1657 
1658 static int xe_oa_set_prop_metric_set(struct xe_oa *oa, u64 value,
1659 				     struct xe_oa_open_param *param)
1660 {
1661 	param->metric_set = value;
1662 	return 0;
1663 }
1664 
1665 static int xe_oa_set_prop_oa_format(struct xe_oa *oa, u64 value,
1666 				    struct xe_oa_open_param *param)
1667 {
1668 	int ret = decode_oa_format(oa, value, &param->oa_format);
1669 
1670 	if (ret) {
1671 		drm_dbg(&oa->xe->drm, "Unsupported OA report format %#llx\n", value);
1672 		return ret;
1673 	}
1674 	return 0;
1675 }
1676 
1677 static int xe_oa_set_prop_oa_exponent(struct xe_oa *oa, u64 value,
1678 				      struct xe_oa_open_param *param)
1679 {
1680 #define OA_EXPONENT_MAX 31
1681 
1682 	if (value > OA_EXPONENT_MAX) {
1683 		drm_dbg(&oa->xe->drm, "OA timer exponent too high (> %u)\n", OA_EXPONENT_MAX);
1684 		return -EINVAL;
1685 	}
1686 	param->period_exponent = value;
1687 	return 0;
1688 }
1689 
1690 static int xe_oa_set_prop_disabled(struct xe_oa *oa, u64 value,
1691 				   struct xe_oa_open_param *param)
1692 {
1693 	param->disabled = value;
1694 	return 0;
1695 }
1696 
1697 static int xe_oa_set_prop_exec_queue_id(struct xe_oa *oa, u64 value,
1698 					struct xe_oa_open_param *param)
1699 {
1700 	param->exec_queue_id = value;
1701 	return 0;
1702 }
1703 
1704 static int xe_oa_set_prop_engine_instance(struct xe_oa *oa, u64 value,
1705 					  struct xe_oa_open_param *param)
1706 {
1707 	param->engine_instance = value;
1708 	return 0;
1709 }
1710 
1711 static int xe_oa_set_no_preempt(struct xe_oa *oa, u64 value,
1712 				struct xe_oa_open_param *param)
1713 {
1714 	param->no_preempt = value;
1715 	return 0;
1716 }
1717 
1718 typedef int (*xe_oa_set_property_fn)(struct xe_oa *oa, u64 value,
1719 				     struct xe_oa_open_param *param);
1720 static const xe_oa_set_property_fn xe_oa_set_property_funcs[] = {
1721 	[DRM_XE_OA_PROPERTY_OA_UNIT_ID] = xe_oa_set_prop_oa_unit_id,
1722 	[DRM_XE_OA_PROPERTY_SAMPLE_OA] = xe_oa_set_prop_sample_oa,
1723 	[DRM_XE_OA_PROPERTY_OA_METRIC_SET] = xe_oa_set_prop_metric_set,
1724 	[DRM_XE_OA_PROPERTY_OA_FORMAT] = xe_oa_set_prop_oa_format,
1725 	[DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT] = xe_oa_set_prop_oa_exponent,
1726 	[DRM_XE_OA_PROPERTY_OA_DISABLED] = xe_oa_set_prop_disabled,
1727 	[DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID] = xe_oa_set_prop_exec_queue_id,
1728 	[DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE] = xe_oa_set_prop_engine_instance,
1729 	[DRM_XE_OA_PROPERTY_NO_PREEMPT] = xe_oa_set_no_preempt,
1730 };
1731 
1732 static int xe_oa_user_ext_set_property(struct xe_oa *oa, u64 extension,
1733 				       struct xe_oa_open_param *param)
1734 {
1735 	u64 __user *address = u64_to_user_ptr(extension);
1736 	struct drm_xe_ext_set_property ext;
1737 	int err;
1738 	u32 idx;
1739 
1740 	err = __copy_from_user(&ext, address, sizeof(ext));
1741 	if (XE_IOCTL_DBG(oa->xe, err))
1742 		return -EFAULT;
1743 
1744 	if (XE_IOCTL_DBG(oa->xe, ext.property >= ARRAY_SIZE(xe_oa_set_property_funcs)) ||
1745 	    XE_IOCTL_DBG(oa->xe, ext.pad))
1746 		return -EINVAL;
1747 
1748 	idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_oa_set_property_funcs));
1749 	return xe_oa_set_property_funcs[idx](oa, ext.value, param);
1750 }
1751 
1752 typedef int (*xe_oa_user_extension_fn)(struct xe_oa *oa, u64 extension,
1753 				       struct xe_oa_open_param *param);
1754 static const xe_oa_user_extension_fn xe_oa_user_extension_funcs[] = {
1755 	[DRM_XE_OA_EXTENSION_SET_PROPERTY] = xe_oa_user_ext_set_property,
1756 };
1757 
1758 #define MAX_USER_EXTENSIONS	16
1759 static int xe_oa_user_extensions(struct xe_oa *oa, u64 extension, int ext_number,
1760 				 struct xe_oa_open_param *param)
1761 {
1762 	u64 __user *address = u64_to_user_ptr(extension);
1763 	struct drm_xe_user_extension ext;
1764 	int err;
1765 	u32 idx;
1766 
1767 	if (XE_IOCTL_DBG(oa->xe, ext_number >= MAX_USER_EXTENSIONS))
1768 		return -E2BIG;
1769 
1770 	err = __copy_from_user(&ext, address, sizeof(ext));
1771 	if (XE_IOCTL_DBG(oa->xe, err))
1772 		return -EFAULT;
1773 
1774 	if (XE_IOCTL_DBG(oa->xe, ext.pad) ||
1775 	    XE_IOCTL_DBG(oa->xe, ext.name >= ARRAY_SIZE(xe_oa_user_extension_funcs)))
1776 		return -EINVAL;
1777 
1778 	idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_oa_user_extension_funcs));
1779 	err = xe_oa_user_extension_funcs[idx](oa, extension, param);
1780 	if (XE_IOCTL_DBG(oa->xe, err))
1781 		return err;
1782 
1783 	if (ext.next_extension)
1784 		return xe_oa_user_extensions(oa, ext.next_extension, ++ext_number, param);
1785 
1786 	return 0;
1787 }
1788 
1789 /**
1790  * xe_oa_stream_open_ioctl - Opens an OA stream
1791  * @dev: @drm_device
1792  * @data: pointer to struct @drm_xe_oa_config
1793  * @file: @drm_file
1794  *
1795  * The functions opens an OA stream. An OA stream, opened with specified
1796  * properties, enables OA counter samples to be collected, either
1797  * periodically (time based sampling), or on request (using OA queries)
1798  */
1799 int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
1800 {
1801 	struct xe_device *xe = to_xe_device(dev);
1802 	struct xe_oa *oa = &xe->oa;
1803 	struct xe_file *xef = to_xe_file(file);
1804 	struct xe_oa_open_param param = {};
1805 	const struct xe_oa_format *f;
1806 	bool privileged_op = true;
1807 	int ret;
1808 
1809 	if (!oa->xe) {
1810 		drm_dbg(&xe->drm, "xe oa interface not available for this system\n");
1811 		return -ENODEV;
1812 	}
1813 
1814 	ret = xe_oa_user_extensions(oa, data, 0, &param);
1815 	if (ret)
1816 		return ret;
1817 
1818 	if (param.exec_queue_id > 0) {
1819 		param.exec_q = xe_exec_queue_lookup(xef, param.exec_queue_id);
1820 		if (XE_IOCTL_DBG(oa->xe, !param.exec_q))
1821 			return -ENOENT;
1822 
1823 		if (param.exec_q->width > 1)
1824 			drm_dbg(&oa->xe->drm, "exec_q->width > 1, programming only exec_q->lrc[0]\n");
1825 	}
1826 
1827 	/*
1828 	 * Query based sampling (using MI_REPORT_PERF_COUNT) with OAR/OAC,
1829 	 * without global stream access, can be an unprivileged operation
1830 	 */
1831 	if (param.exec_q && !param.sample)
1832 		privileged_op = false;
1833 
1834 	if (param.no_preempt) {
1835 		if (!param.exec_q) {
1836 			drm_dbg(&oa->xe->drm, "Preemption disable without exec_q!\n");
1837 			ret = -EINVAL;
1838 			goto err_exec_q;
1839 		}
1840 		privileged_op = true;
1841 	}
1842 
1843 	if (privileged_op && xe_observation_paranoid && !perfmon_capable()) {
1844 		drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n");
1845 		ret = -EACCES;
1846 		goto err_exec_q;
1847 	}
1848 
1849 	if (!param.exec_q && !param.sample) {
1850 		drm_dbg(&oa->xe->drm, "Only OA report sampling supported\n");
1851 		ret = -EINVAL;
1852 		goto err_exec_q;
1853 	}
1854 
1855 	ret = xe_oa_assign_hwe(oa, &param);
1856 	if (ret)
1857 		goto err_exec_q;
1858 
1859 	f = &oa->oa_formats[param.oa_format];
1860 	if (!param.oa_format || !f->size ||
1861 	    !engine_supports_oa_format(param.hwe, f->type)) {
1862 		drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n",
1863 			param.oa_format, f->type, f->size, param.hwe->class);
1864 		ret = -EINVAL;
1865 		goto err_exec_q;
1866 	}
1867 
1868 	if (param.period_exponent > 0) {
1869 		u64 oa_period, oa_freq_hz;
1870 
1871 		/* Requesting samples from OAG buffer is a privileged operation */
1872 		if (!param.sample) {
1873 			drm_dbg(&oa->xe->drm, "OA_EXPONENT specified without SAMPLE_OA\n");
1874 			ret = -EINVAL;
1875 			goto err_exec_q;
1876 		}
1877 		oa_period = oa_exponent_to_ns(param.hwe->gt, param.period_exponent);
1878 		oa_freq_hz = div64_u64(NSEC_PER_SEC, oa_period);
1879 		drm_dbg(&oa->xe->drm, "Using periodic sampling freq %lld Hz\n", oa_freq_hz);
1880 	}
1881 
1882 	mutex_lock(&param.hwe->gt->oa.gt_lock);
1883 	ret = xe_oa_stream_open_ioctl_locked(oa, &param);
1884 	mutex_unlock(&param.hwe->gt->oa.gt_lock);
1885 err_exec_q:
1886 	if (ret < 0 && param.exec_q)
1887 		xe_exec_queue_put(param.exec_q);
1888 	return ret;
1889 }
1890 
1891 static bool xe_oa_is_valid_flex_addr(struct xe_oa *oa, u32 addr)
1892 {
1893 	static const struct xe_reg flex_eu_regs[] = {
1894 		EU_PERF_CNTL0,
1895 		EU_PERF_CNTL1,
1896 		EU_PERF_CNTL2,
1897 		EU_PERF_CNTL3,
1898 		EU_PERF_CNTL4,
1899 		EU_PERF_CNTL5,
1900 		EU_PERF_CNTL6,
1901 	};
1902 	int i;
1903 
1904 	for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) {
1905 		if (flex_eu_regs[i].addr == addr)
1906 			return true;
1907 	}
1908 	return false;
1909 }
1910 
1911 static bool xe_oa_reg_in_range_table(u32 addr, const struct xe_mmio_range *table)
1912 {
1913 	while (table->start && table->end) {
1914 		if (addr >= table->start && addr <= table->end)
1915 			return true;
1916 
1917 		table++;
1918 	}
1919 
1920 	return false;
1921 }
1922 
1923 static const struct xe_mmio_range xehp_oa_b_counters[] = {
1924 	{ .start = 0xdc48, .end = 0xdc48 },	/* OAA_ENABLE_REG */
1925 	{ .start = 0xdd00, .end = 0xdd48 },	/* OAG_LCE0_0 - OAA_LENABLE_REG */
1926 	{}
1927 };
1928 
1929 static const struct xe_mmio_range gen12_oa_b_counters[] = {
1930 	{ .start = 0x2b2c, .end = 0x2b2c },	/* OAG_OA_PESS */
1931 	{ .start = 0xd900, .end = 0xd91c },	/* OAG_OASTARTTRIG[1-8] */
1932 	{ .start = 0xd920, .end = 0xd93c },	/* OAG_OAREPORTTRIG1[1-8] */
1933 	{ .start = 0xd940, .end = 0xd97c },	/* OAG_CEC[0-7][0-1] */
1934 	{ .start = 0xdc00, .end = 0xdc3c },	/* OAG_SCEC[0-7][0-1] */
1935 	{ .start = 0xdc40, .end = 0xdc40 },	/* OAG_SPCTR_CNF */
1936 	{ .start = 0xdc44, .end = 0xdc44 },	/* OAA_DBG_REG */
1937 	{}
1938 };
1939 
1940 static const struct xe_mmio_range mtl_oam_b_counters[] = {
1941 	{ .start = 0x393000, .end = 0x39301c },	/* OAM_STARTTRIG1[1-8] */
1942 	{ .start = 0x393020, .end = 0x39303c },	/* OAM_REPORTTRIG1[1-8] */
1943 	{ .start = 0x393040, .end = 0x39307c },	/* OAM_CEC[0-7][0-1] */
1944 	{ .start = 0x393200, .end = 0x39323C },	/* MPES[0-7] */
1945 	{}
1946 };
1947 
1948 static const struct xe_mmio_range xe2_oa_b_counters[] = {
1949 	{ .start = 0x393200, .end = 0x39323C },	/* MPES_0_MPES_SAG - MPES_7_UPPER_MPES_SAG */
1950 	{ .start = 0x394200, .end = 0x39423C },	/* MPES_0_MPES_SCMI0 - MPES_7_UPPER_MPES_SCMI0 */
1951 	{ .start = 0x394A00, .end = 0x394A3C },	/* MPES_0_MPES_SCMI1 - MPES_7_UPPER_MPES_SCMI1 */
1952 	{},
1953 };
1954 
1955 static bool xe_oa_is_valid_b_counter_addr(struct xe_oa *oa, u32 addr)
1956 {
1957 	return xe_oa_reg_in_range_table(addr, xehp_oa_b_counters) ||
1958 		xe_oa_reg_in_range_table(addr, gen12_oa_b_counters) ||
1959 		xe_oa_reg_in_range_table(addr, mtl_oam_b_counters) ||
1960 		(GRAPHICS_VER(oa->xe) >= 20 &&
1961 		 xe_oa_reg_in_range_table(addr, xe2_oa_b_counters));
1962 }
1963 
1964 static const struct xe_mmio_range mtl_oa_mux_regs[] = {
1965 	{ .start = 0x0d00, .end = 0x0d04 },	/* RPM_CONFIG[0-1] */
1966 	{ .start = 0x0d0c, .end = 0x0d2c },	/* NOA_CONFIG[0-8] */
1967 	{ .start = 0x9840, .end = 0x9840 },	/* GDT_CHICKEN_BITS */
1968 	{ .start = 0x9884, .end = 0x9888 },	/* NOA_WRITE */
1969 	{ .start = 0x38d100, .end = 0x38d114},	/* VISACTL */
1970 	{}
1971 };
1972 
1973 static const struct xe_mmio_range gen12_oa_mux_regs[] = {
1974 	{ .start = 0x0d00, .end = 0x0d04 },     /* RPM_CONFIG[0-1] */
1975 	{ .start = 0x0d0c, .end = 0x0d2c },     /* NOA_CONFIG[0-8] */
1976 	{ .start = 0x9840, .end = 0x9840 },	/* GDT_CHICKEN_BITS */
1977 	{ .start = 0x9884, .end = 0x9888 },	/* NOA_WRITE */
1978 	{ .start = 0x20cc, .end = 0x20cc },	/* WAIT_FOR_RC6_EXIT */
1979 	{}
1980 };
1981 
1982 static const struct xe_mmio_range xe2_oa_mux_regs[] = {
1983 	{ .start = 0x5194, .end = 0x5194 },	/* SYS_MEM_LAT_MEASURE_MERTF_GRP_3D */
1984 	{ .start = 0x8704, .end = 0x8704 },	/* LMEM_LAT_MEASURE_MCFG_GRP */
1985 	{ .start = 0xB1BC, .end = 0xB1BC },	/* L3_BANK_LAT_MEASURE_LBCF_GFX */
1986 	{ .start = 0xE18C, .end = 0xE18C },	/* SAMPLER_MODE */
1987 	{ .start = 0xE590, .end = 0xE590 },	/* TDL_LSC_LAT_MEASURE_TDL_GFX */
1988 	{ .start = 0x13000, .end = 0x137FC },	/* PES_0_PESL0 - PES_63_UPPER_PESL3 */
1989 	{},
1990 };
1991 
1992 static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr)
1993 {
1994 	if (GRAPHICS_VER(oa->xe) >= 20)
1995 		return xe_oa_reg_in_range_table(addr, xe2_oa_mux_regs);
1996 	else if (GRAPHICS_VERx100(oa->xe) >= 1270)
1997 		return xe_oa_reg_in_range_table(addr, mtl_oa_mux_regs);
1998 	else
1999 		return xe_oa_reg_in_range_table(addr, gen12_oa_mux_regs);
2000 }
2001 
2002 static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr)
2003 {
2004 	return xe_oa_is_valid_flex_addr(oa, addr) ||
2005 		xe_oa_is_valid_b_counter_addr(oa, addr) ||
2006 		xe_oa_is_valid_mux_addr(oa, addr);
2007 }
2008 
2009 static struct xe_oa_reg *
2010 xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr),
2011 		 u32 __user *regs, u32 n_regs)
2012 {
2013 	struct xe_oa_reg *oa_regs;
2014 	int err;
2015 	u32 i;
2016 
2017 	oa_regs = kmalloc_array(n_regs, sizeof(*oa_regs), GFP_KERNEL);
2018 	if (!oa_regs)
2019 		return ERR_PTR(-ENOMEM);
2020 
2021 	for (i = 0; i < n_regs; i++) {
2022 		u32 addr, value;
2023 
2024 		err = get_user(addr, regs);
2025 		if (err)
2026 			goto addr_err;
2027 
2028 		if (!is_valid(oa, addr)) {
2029 			drm_dbg(&oa->xe->drm, "Invalid oa_reg address: %X\n", addr);
2030 			err = -EINVAL;
2031 			goto addr_err;
2032 		}
2033 
2034 		err = get_user(value, regs + 1);
2035 		if (err)
2036 			goto addr_err;
2037 
2038 		oa_regs[i].addr = XE_REG(addr);
2039 		oa_regs[i].value = value;
2040 
2041 		regs += 2;
2042 	}
2043 
2044 	return oa_regs;
2045 
2046 addr_err:
2047 	kfree(oa_regs);
2048 	return ERR_PTR(err);
2049 }
2050 
2051 static ssize_t show_dynamic_id(struct kobject *kobj,
2052 			       struct kobj_attribute *attr,
2053 			       char *buf)
2054 {
2055 	struct xe_oa_config *oa_config =
2056 		container_of(attr, typeof(*oa_config), sysfs_metric_id);
2057 
2058 	return sysfs_emit(buf, "%d\n", oa_config->id);
2059 }
2060 
2061 static int create_dynamic_oa_sysfs_entry(struct xe_oa *oa,
2062 					 struct xe_oa_config *oa_config)
2063 {
2064 	sysfs_attr_init(&oa_config->sysfs_metric_id.attr);
2065 	oa_config->sysfs_metric_id.attr.name = "id";
2066 	oa_config->sysfs_metric_id.attr.mode = 0444;
2067 	oa_config->sysfs_metric_id.show = show_dynamic_id;
2068 	oa_config->sysfs_metric_id.store = NULL;
2069 
2070 	oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr;
2071 	oa_config->attrs[1] = NULL;
2072 
2073 	oa_config->sysfs_metric.name = oa_config->uuid;
2074 	oa_config->sysfs_metric.attrs = oa_config->attrs;
2075 
2076 	return sysfs_create_group(oa->metrics_kobj, &oa_config->sysfs_metric);
2077 }
2078 
2079 /**
2080  * xe_oa_add_config_ioctl - Adds one OA config
2081  * @dev: @drm_device
2082  * @data: pointer to struct @drm_xe_oa_config
2083  * @file: @drm_file
2084  *
2085  * The functions adds an OA config to the set of OA configs maintained in
2086  * the kernel. The config determines which OA metrics are collected for an
2087  * OA stream.
2088  */
2089 int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
2090 {
2091 	struct xe_device *xe = to_xe_device(dev);
2092 	struct xe_oa *oa = &xe->oa;
2093 	struct drm_xe_oa_config param;
2094 	struct drm_xe_oa_config *arg = &param;
2095 	struct xe_oa_config *oa_config, *tmp;
2096 	struct xe_oa_reg *regs;
2097 	int err, id;
2098 
2099 	if (!oa->xe) {
2100 		drm_dbg(&xe->drm, "xe oa interface not available for this system\n");
2101 		return -ENODEV;
2102 	}
2103 
2104 	if (xe_observation_paranoid && !perfmon_capable()) {
2105 		drm_dbg(&oa->xe->drm, "Insufficient privileges to add xe OA config\n");
2106 		return -EACCES;
2107 	}
2108 
2109 	err = __copy_from_user(&param, u64_to_user_ptr(data), sizeof(param));
2110 	if (XE_IOCTL_DBG(oa->xe, err))
2111 		return -EFAULT;
2112 
2113 	if (XE_IOCTL_DBG(oa->xe, arg->extensions) ||
2114 	    XE_IOCTL_DBG(oa->xe, !arg->regs_ptr) ||
2115 	    XE_IOCTL_DBG(oa->xe, !arg->n_regs))
2116 		return -EINVAL;
2117 
2118 	oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL);
2119 	if (!oa_config)
2120 		return -ENOMEM;
2121 
2122 	oa_config->oa = oa;
2123 	kref_init(&oa_config->ref);
2124 
2125 	if (!uuid_is_valid(arg->uuid)) {
2126 		drm_dbg(&oa->xe->drm, "Invalid uuid format for OA config\n");
2127 		err = -EINVAL;
2128 		goto reg_err;
2129 	}
2130 
2131 	/* Last character in oa_config->uuid will be 0 because oa_config is kzalloc */
2132 	memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid));
2133 
2134 	oa_config->regs_len = arg->n_regs;
2135 	regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg_addr,
2136 				u64_to_user_ptr(arg->regs_ptr),
2137 				arg->n_regs);
2138 	if (IS_ERR(regs)) {
2139 		drm_dbg(&oa->xe->drm, "Failed to create OA config for mux_regs\n");
2140 		err = PTR_ERR(regs);
2141 		goto reg_err;
2142 	}
2143 	oa_config->regs = regs;
2144 
2145 	err = mutex_lock_interruptible(&oa->metrics_lock);
2146 	if (err)
2147 		goto reg_err;
2148 
2149 	/* We shouldn't have too many configs, so this iteration shouldn't be too costly */
2150 	idr_for_each_entry(&oa->metrics_idr, tmp, id) {
2151 		if (!strcmp(tmp->uuid, oa_config->uuid)) {
2152 			drm_dbg(&oa->xe->drm, "OA config already exists with this uuid\n");
2153 			err = -EADDRINUSE;
2154 			goto sysfs_err;
2155 		}
2156 	}
2157 
2158 	err = create_dynamic_oa_sysfs_entry(oa, oa_config);
2159 	if (err) {
2160 		drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n");
2161 		goto sysfs_err;
2162 	}
2163 
2164 	oa_config->id = idr_alloc(&oa->metrics_idr, oa_config, 1, 0, GFP_KERNEL);
2165 	if (oa_config->id < 0) {
2166 		drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n");
2167 		err = oa_config->id;
2168 		goto sysfs_err;
2169 	}
2170 
2171 	mutex_unlock(&oa->metrics_lock);
2172 
2173 	drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, oa_config->id);
2174 
2175 	return oa_config->id;
2176 
2177 sysfs_err:
2178 	mutex_unlock(&oa->metrics_lock);
2179 reg_err:
2180 	xe_oa_config_put(oa_config);
2181 	drm_dbg(&oa->xe->drm, "Failed to add new OA config\n");
2182 	return err;
2183 }
2184 
2185 /**
2186  * xe_oa_remove_config_ioctl - Removes one OA config
2187  * @dev: @drm_device
2188  * @data: pointer to struct @drm_xe_observation_param
2189  * @file: @drm_file
2190  */
2191 int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
2192 {
2193 	struct xe_device *xe = to_xe_device(dev);
2194 	struct xe_oa *oa = &xe->oa;
2195 	struct xe_oa_config *oa_config;
2196 	u64 arg, *ptr = u64_to_user_ptr(data);
2197 	int ret;
2198 
2199 	if (!oa->xe) {
2200 		drm_dbg(&xe->drm, "xe oa interface not available for this system\n");
2201 		return -ENODEV;
2202 	}
2203 
2204 	if (xe_observation_paranoid && !perfmon_capable()) {
2205 		drm_dbg(&oa->xe->drm, "Insufficient privileges to remove xe OA config\n");
2206 		return -EACCES;
2207 	}
2208 
2209 	ret = get_user(arg, ptr);
2210 	if (XE_IOCTL_DBG(oa->xe, ret))
2211 		return ret;
2212 
2213 	ret = mutex_lock_interruptible(&oa->metrics_lock);
2214 	if (ret)
2215 		return ret;
2216 
2217 	oa_config = idr_find(&oa->metrics_idr, arg);
2218 	if (!oa_config) {
2219 		drm_dbg(&oa->xe->drm, "Failed to remove unknown OA config\n");
2220 		ret = -ENOENT;
2221 		goto err_unlock;
2222 	}
2223 
2224 	WARN_ON(arg != oa_config->id);
2225 
2226 	sysfs_remove_group(oa->metrics_kobj, &oa_config->sysfs_metric);
2227 	idr_remove(&oa->metrics_idr, arg);
2228 
2229 	mutex_unlock(&oa->metrics_lock);
2230 
2231 	drm_dbg(&oa->xe->drm, "Removed config %s id=%i\n", oa_config->uuid, oa_config->id);
2232 
2233 	xe_oa_config_put(oa_config);
2234 
2235 	return 0;
2236 
2237 err_unlock:
2238 	mutex_unlock(&oa->metrics_lock);
2239 	return ret;
2240 }
2241 
2242 /**
2243  * xe_oa_register - Xe OA registration
2244  * @xe: @xe_device
2245  *
2246  * Exposes the metrics sysfs directory upon completion of module initialization
2247  */
2248 void xe_oa_register(struct xe_device *xe)
2249 {
2250 	struct xe_oa *oa = &xe->oa;
2251 
2252 	if (!oa->xe)
2253 		return;
2254 
2255 	oa->metrics_kobj = kobject_create_and_add("metrics",
2256 						  &xe->drm.primary->kdev->kobj);
2257 }
2258 
2259 /**
2260  * xe_oa_unregister - Xe OA de-registration
2261  * @xe: @xe_device
2262  */
2263 void xe_oa_unregister(struct xe_device *xe)
2264 {
2265 	struct xe_oa *oa = &xe->oa;
2266 
2267 	if (!oa->metrics_kobj)
2268 		return;
2269 
2270 	kobject_put(oa->metrics_kobj);
2271 	oa->metrics_kobj = NULL;
2272 }
2273 
2274 static u32 num_oa_units_per_gt(struct xe_gt *gt)
2275 {
2276 	return 1;
2277 }
2278 
2279 static u32 __hwe_oam_unit(struct xe_hw_engine *hwe)
2280 {
2281 	if (GRAPHICS_VERx100(gt_to_xe(hwe->gt)) >= 1270) {
2282 		/*
2283 		 * There's 1 SAMEDIA gt and 1 OAM per SAMEDIA gt. All media slices
2284 		 * within the gt use the same OAM. All MTL/LNL SKUs list 1 SA MEDIA
2285 		 */
2286 		xe_gt_WARN_ON(hwe->gt, hwe->gt->info.type != XE_GT_TYPE_MEDIA);
2287 
2288 		return 0;
2289 	}
2290 
2291 	return XE_OA_UNIT_INVALID;
2292 }
2293 
2294 static u32 __hwe_oa_unit(struct xe_hw_engine *hwe)
2295 {
2296 	switch (hwe->class) {
2297 	case XE_ENGINE_CLASS_RENDER:
2298 	case XE_ENGINE_CLASS_COMPUTE:
2299 		return 0;
2300 
2301 	case XE_ENGINE_CLASS_VIDEO_DECODE:
2302 	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
2303 		return __hwe_oam_unit(hwe);
2304 
2305 	default:
2306 		return XE_OA_UNIT_INVALID;
2307 	}
2308 }
2309 
2310 static struct xe_oa_regs __oam_regs(u32 base)
2311 {
2312 	return (struct xe_oa_regs) {
2313 		base,
2314 		OAM_HEAD_POINTER(base),
2315 		OAM_TAIL_POINTER(base),
2316 		OAM_BUFFER(base),
2317 		OAM_CONTEXT_CONTROL(base),
2318 		OAM_CONTROL(base),
2319 		OAM_DEBUG(base),
2320 		OAM_STATUS(base),
2321 		OAM_CONTROL_COUNTER_SEL_MASK,
2322 	};
2323 }
2324 
2325 static struct xe_oa_regs __oag_regs(void)
2326 {
2327 	return (struct xe_oa_regs) {
2328 		0,
2329 		OAG_OAHEADPTR,
2330 		OAG_OATAILPTR,
2331 		OAG_OABUFFER,
2332 		OAG_OAGLBCTXCTRL,
2333 		OAG_OACONTROL,
2334 		OAG_OA_DEBUG,
2335 		OAG_OASTATUS,
2336 		OAG_OACONTROL_OA_COUNTER_SEL_MASK,
2337 	};
2338 }
2339 
2340 static void __xe_oa_init_oa_units(struct xe_gt *gt)
2341 {
2342 	const u32 mtl_oa_base[] = { 0x13000 };
2343 	int i, num_units = gt->oa.num_oa_units;
2344 
2345 	for (i = 0; i < num_units; i++) {
2346 		struct xe_oa_unit *u = &gt->oa.oa_unit[i];
2347 
2348 		if (gt->info.type != XE_GT_TYPE_MEDIA) {
2349 			u->regs = __oag_regs();
2350 			u->type = DRM_XE_OA_UNIT_TYPE_OAG;
2351 		} else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
2352 			u->regs = __oam_regs(mtl_oa_base[i]);
2353 			u->type = DRM_XE_OA_UNIT_TYPE_OAM;
2354 		}
2355 
2356 		/* Ensure MMIO trigger remains disabled till there is a stream */
2357 		xe_mmio_write32(gt, u->regs.oa_debug,
2358 				oag_configure_mmio_trigger(NULL, false));
2359 
2360 		/* Set oa_unit_ids now to ensure ids remain contiguous */
2361 		u->oa_unit_id = gt_to_xe(gt)->oa.oa_unit_ids++;
2362 	}
2363 }
2364 
2365 static int xe_oa_init_gt(struct xe_gt *gt)
2366 {
2367 	u32 num_oa_units = num_oa_units_per_gt(gt);
2368 	struct xe_hw_engine *hwe;
2369 	enum xe_hw_engine_id id;
2370 	struct xe_oa_unit *u;
2371 
2372 	u = drmm_kcalloc(&gt_to_xe(gt)->drm, num_oa_units, sizeof(*u), GFP_KERNEL);
2373 	if (!u)
2374 		return -ENOMEM;
2375 
2376 	for_each_hw_engine(hwe, gt, id) {
2377 		u32 index = __hwe_oa_unit(hwe);
2378 
2379 		hwe->oa_unit = NULL;
2380 		if (index < num_oa_units) {
2381 			u[index].num_engines++;
2382 			hwe->oa_unit = &u[index];
2383 		}
2384 	}
2385 
2386 	/*
2387 	 * Fused off engines can result in oa_unit's with num_engines == 0. These units
2388 	 * will appear in OA unit query, but no OA streams can be opened on them.
2389 	 */
2390 	gt->oa.num_oa_units = num_oa_units;
2391 	gt->oa.oa_unit = u;
2392 
2393 	__xe_oa_init_oa_units(gt);
2394 
2395 	drmm_mutex_init(&gt_to_xe(gt)->drm, &gt->oa.gt_lock);
2396 
2397 	return 0;
2398 }
2399 
2400 static int xe_oa_init_oa_units(struct xe_oa *oa)
2401 {
2402 	struct xe_gt *gt;
2403 	int i, ret;
2404 
2405 	for_each_gt(gt, oa->xe, i) {
2406 		ret = xe_oa_init_gt(gt);
2407 		if (ret)
2408 			return ret;
2409 	}
2410 
2411 	return 0;
2412 }
2413 
2414 static void oa_format_add(struct xe_oa *oa, enum xe_oa_format_name format)
2415 {
2416 	__set_bit(format, oa->format_mask);
2417 }
2418 
2419 static void xe_oa_init_supported_formats(struct xe_oa *oa)
2420 {
2421 	if (GRAPHICS_VER(oa->xe) >= 20) {
2422 		/* Xe2+ */
2423 		oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8);
2424 		oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8);
2425 		oa_format_add(oa, XE_OA_FORMAT_PEC64u64);
2426 		oa_format_add(oa, XE_OA_FORMAT_PEC64u64_B8_C8);
2427 		oa_format_add(oa, XE_OA_FORMAT_PEC64u32);
2428 		oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G1);
2429 		oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G1);
2430 		oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G2);
2431 		oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G2);
2432 		oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_32_G2_4);
2433 		oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_4_G2_32);
2434 	} else if (GRAPHICS_VERx100(oa->xe) >= 1270) {
2435 		/* XE_METEORLAKE */
2436 		oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8);
2437 		oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8);
2438 		oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8);
2439 		oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8);
2440 		oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8);
2441 		oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8);
2442 	} else if (GRAPHICS_VERx100(oa->xe) >= 1255) {
2443 		/* XE_DG2, XE_PVC */
2444 		oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8);
2445 		oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8);
2446 		oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8);
2447 		oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8);
2448 	} else {
2449 		/* Gen12+ */
2450 		xe_assert(oa->xe, GRAPHICS_VER(oa->xe) >= 12);
2451 		oa_format_add(oa, XE_OA_FORMAT_A12);
2452 		oa_format_add(oa, XE_OA_FORMAT_A12_B8_C8);
2453 		oa_format_add(oa, XE_OA_FORMAT_A32u40_A4u32_B8_C8);
2454 		oa_format_add(oa, XE_OA_FORMAT_C4_B8);
2455 	}
2456 }
2457 
2458 /**
2459  * xe_oa_init - OA initialization during device probe
2460  * @xe: @xe_device
2461  *
2462  * Return: 0 on success or a negative error code on failure
2463  */
2464 int xe_oa_init(struct xe_device *xe)
2465 {
2466 	struct xe_oa *oa = &xe->oa;
2467 	int ret;
2468 
2469 	/* Support OA only with GuC submission and Gen12+ */
2470 	if (!xe_device_uc_enabled(xe) || GRAPHICS_VER(xe) < 12)
2471 		return 0;
2472 
2473 	if (IS_SRIOV_VF(xe))
2474 		return 0;
2475 
2476 	oa->xe = xe;
2477 	oa->oa_formats = oa_formats;
2478 
2479 	drmm_mutex_init(&oa->xe->drm, &oa->metrics_lock);
2480 	idr_init_base(&oa->metrics_idr, 1);
2481 
2482 	ret = xe_oa_init_oa_units(oa);
2483 	if (ret) {
2484 		drm_err(&xe->drm, "OA initialization failed (%pe)\n", ERR_PTR(ret));
2485 		goto exit;
2486 	}
2487 
2488 	xe_oa_init_supported_formats(oa);
2489 	return 0;
2490 exit:
2491 	oa->xe = NULL;
2492 	return ret;
2493 }
2494 
2495 static int destroy_config(int id, void *p, void *data)
2496 {
2497 	xe_oa_config_put(p);
2498 	return 0;
2499 }
2500 
2501 /**
2502  * xe_oa_fini - OA de-initialization during device remove
2503  * @xe: @xe_device
2504  */
2505 void xe_oa_fini(struct xe_device *xe)
2506 {
2507 	struct xe_oa *oa = &xe->oa;
2508 
2509 	if (!oa->xe)
2510 		return;
2511 
2512 	idr_for_each(&oa->metrics_idr, destroy_config, oa);
2513 	idr_destroy(&oa->metrics_idr);
2514 
2515 	oa->xe = NULL;
2516 }
2517