xref: /linux/drivers/gpu/drm/xe/xe_oa.c (revision 76544811c850a1f4c055aa182b513b7a843868ea)
1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2023-2024 Intel Corporation
4  */
5 
6 #include <linux/anon_inodes.h>
7 #include <linux/delay.h>
8 #include <linux/nospec.h>
9 #include <linux/poll.h>
10 
11 #include <drm/drm_drv.h>
12 #include <drm/drm_managed.h>
13 #include <uapi/drm/xe_drm.h>
14 
15 #include "abi/guc_actions_slpc_abi.h"
16 #include "instructions/xe_mi_commands.h"
17 #include "regs/xe_engine_regs.h"
18 #include "regs/xe_gt_regs.h"
19 #include "regs/xe_oa_regs.h"
20 #include "xe_assert.h"
21 #include "xe_bb.h"
22 #include "xe_bo.h"
23 #include "xe_device.h"
24 #include "xe_exec_queue.h"
25 #include "xe_force_wake.h"
26 #include "xe_gt.h"
27 #include "xe_gt_mcr.h"
28 #include "xe_gt_printk.h"
29 #include "xe_guc_pc.h"
30 #include "xe_macros.h"
31 #include "xe_mmio.h"
32 #include "xe_oa.h"
33 #include "xe_observation.h"
34 #include "xe_pm.h"
35 #include "xe_sched_job.h"
36 #include "xe_sriov.h"
37 #include "xe_sync.h"
38 
39 #define DEFAULT_POLL_FREQUENCY_HZ 200
40 #define DEFAULT_POLL_PERIOD_NS (NSEC_PER_SEC / DEFAULT_POLL_FREQUENCY_HZ)
41 #define XE_OA_UNIT_INVALID U32_MAX
42 
43 enum xe_oa_submit_deps {
44 	XE_OA_SUBMIT_NO_DEPS,
45 	XE_OA_SUBMIT_ADD_DEPS,
46 };
47 
48 enum xe_oa_user_extn_from {
49 	XE_OA_USER_EXTN_FROM_OPEN,
50 	XE_OA_USER_EXTN_FROM_CONFIG,
51 };
52 
53 struct xe_oa_reg {
54 	struct xe_reg addr;
55 	u32 value;
56 };
57 
58 struct xe_oa_config {
59 	struct xe_oa *oa;
60 
61 	char uuid[UUID_STRING_LEN + 1];
62 	int id;
63 
64 	const struct xe_oa_reg *regs;
65 	u32 regs_len;
66 
67 	struct attribute_group sysfs_metric;
68 	struct attribute *attrs[2];
69 	struct kobj_attribute sysfs_metric_id;
70 
71 	struct kref ref;
72 	struct rcu_head rcu;
73 };
74 
75 struct xe_oa_open_param {
76 	struct xe_file *xef;
77 	u32 oa_unit_id;
78 	bool sample;
79 	u32 metric_set;
80 	enum xe_oa_format_name oa_format;
81 	int period_exponent;
82 	bool disabled;
83 	int exec_queue_id;
84 	int engine_instance;
85 	struct xe_exec_queue *exec_q;
86 	struct xe_hw_engine *hwe;
87 	bool no_preempt;
88 	struct drm_xe_sync __user *syncs_user;
89 	int num_syncs;
90 	struct xe_sync_entry *syncs;
91 	size_t oa_buffer_size;
92 	int wait_num_reports;
93 };
94 
95 struct xe_oa_config_bo {
96 	struct llist_node node;
97 
98 	struct xe_oa_config *oa_config;
99 	struct xe_bb *bb;
100 };
101 
102 struct xe_oa_fence {
103 	/* @base: dma fence base */
104 	struct dma_fence base;
105 	/* @lock: lock for the fence */
106 	spinlock_t lock;
107 	/* @work: work to signal @base */
108 	struct delayed_work work;
109 	/* @cb: callback to schedule @work */
110 	struct dma_fence_cb cb;
111 };
112 
113 #define DRM_FMT(x) DRM_XE_OA_FMT_TYPE_##x
114 
115 static const struct xe_oa_format oa_formats[] = {
116 	[XE_OA_FORMAT_C4_B8]			= { 7, 64,  DRM_FMT(OAG) },
117 	[XE_OA_FORMAT_A12]			= { 0, 64,  DRM_FMT(OAG) },
118 	[XE_OA_FORMAT_A12_B8_C8]		= { 2, 128, DRM_FMT(OAG) },
119 	[XE_OA_FORMAT_A32u40_A4u32_B8_C8]	= { 5, 256, DRM_FMT(OAG) },
120 	[XE_OAR_FORMAT_A32u40_A4u32_B8_C8]	= { 5, 256, DRM_FMT(OAR) },
121 	[XE_OA_FORMAT_A24u40_A14u32_B8_C8]	= { 5, 256, DRM_FMT(OAG) },
122 	[XE_OAC_FORMAT_A24u64_B8_C8]		= { 1, 320, DRM_FMT(OAC), HDR_64_BIT },
123 	[XE_OAC_FORMAT_A22u32_R2u32_B8_C8]	= { 2, 192, DRM_FMT(OAC), HDR_64_BIT },
124 	[XE_OAM_FORMAT_MPEC8u64_B8_C8]		= { 1, 192, DRM_FMT(OAM_MPEC), HDR_64_BIT },
125 	[XE_OAM_FORMAT_MPEC8u32_B8_C8]		= { 2, 128, DRM_FMT(OAM_MPEC), HDR_64_BIT },
126 	[XE_OA_FORMAT_PEC64u64]			= { 1, 576, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
127 	[XE_OA_FORMAT_PEC64u64_B8_C8]		= { 1, 640, DRM_FMT(PEC), HDR_64_BIT, 1, 1 },
128 	[XE_OA_FORMAT_PEC64u32]			= { 1, 320, DRM_FMT(PEC), HDR_64_BIT },
129 	[XE_OA_FORMAT_PEC32u64_G1]		= { 5, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
130 	[XE_OA_FORMAT_PEC32u32_G1]		= { 5, 192, DRM_FMT(PEC), HDR_64_BIT },
131 	[XE_OA_FORMAT_PEC32u64_G2]		= { 6, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
132 	[XE_OA_FORMAT_PEC32u32_G2]		= { 6, 192, DRM_FMT(PEC), HDR_64_BIT },
133 	[XE_OA_FORMAT_PEC36u64_G1_32_G2_4]	= { 3, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
134 	[XE_OA_FORMAT_PEC36u64_G1_4_G2_32]	= { 4, 320, DRM_FMT(PEC), HDR_64_BIT, 1, 0 },
135 };
136 
xe_oa_circ_diff(struct xe_oa_stream * stream,u32 tail,u32 head)137 static u32 xe_oa_circ_diff(struct xe_oa_stream *stream, u32 tail, u32 head)
138 {
139 	return tail >= head ? tail - head :
140 		tail + stream->oa_buffer.circ_size - head;
141 }
142 
xe_oa_circ_incr(struct xe_oa_stream * stream,u32 ptr,u32 n)143 static u32 xe_oa_circ_incr(struct xe_oa_stream *stream, u32 ptr, u32 n)
144 {
145 	return ptr + n >= stream->oa_buffer.circ_size ?
146 		ptr + n - stream->oa_buffer.circ_size : ptr + n;
147 }
148 
xe_oa_config_release(struct kref * ref)149 static void xe_oa_config_release(struct kref *ref)
150 {
151 	struct xe_oa_config *oa_config =
152 		container_of(ref, typeof(*oa_config), ref);
153 
154 	kfree(oa_config->regs);
155 
156 	kfree_rcu(oa_config, rcu);
157 }
158 
xe_oa_config_put(struct xe_oa_config * oa_config)159 static void xe_oa_config_put(struct xe_oa_config *oa_config)
160 {
161 	if (!oa_config)
162 		return;
163 
164 	kref_put(&oa_config->ref, xe_oa_config_release);
165 }
166 
xe_oa_config_get(struct xe_oa_config * oa_config)167 static struct xe_oa_config *xe_oa_config_get(struct xe_oa_config *oa_config)
168 {
169 	return kref_get_unless_zero(&oa_config->ref) ? oa_config : NULL;
170 }
171 
xe_oa_get_oa_config(struct xe_oa * oa,int metrics_set)172 static struct xe_oa_config *xe_oa_get_oa_config(struct xe_oa *oa, int metrics_set)
173 {
174 	struct xe_oa_config *oa_config;
175 
176 	rcu_read_lock();
177 	oa_config = idr_find(&oa->metrics_idr, metrics_set);
178 	if (oa_config)
179 		oa_config = xe_oa_config_get(oa_config);
180 	rcu_read_unlock();
181 
182 	return oa_config;
183 }
184 
free_oa_config_bo(struct xe_oa_config_bo * oa_bo,struct dma_fence * last_fence)185 static void free_oa_config_bo(struct xe_oa_config_bo *oa_bo, struct dma_fence *last_fence)
186 {
187 	xe_oa_config_put(oa_bo->oa_config);
188 	xe_bb_free(oa_bo->bb, last_fence);
189 	kfree(oa_bo);
190 }
191 
__oa_regs(struct xe_oa_stream * stream)192 static const struct xe_oa_regs *__oa_regs(struct xe_oa_stream *stream)
193 {
194 	return &stream->hwe->oa_unit->regs;
195 }
196 
xe_oa_hw_tail_read(struct xe_oa_stream * stream)197 static u32 xe_oa_hw_tail_read(struct xe_oa_stream *stream)
198 {
199 	return xe_mmio_read32(&stream->gt->mmio, __oa_regs(stream)->oa_tail_ptr) &
200 		OAG_OATAILPTR_MASK;
201 }
202 
203 #define oa_report_header_64bit(__s) \
204 	((__s)->oa_buffer.format->header == HDR_64_BIT)
205 
oa_report_id(struct xe_oa_stream * stream,void * report)206 static u64 oa_report_id(struct xe_oa_stream *stream, void *report)
207 {
208 	return oa_report_header_64bit(stream) ? *(u64 *)report : *(u32 *)report;
209 }
210 
oa_report_id_clear(struct xe_oa_stream * stream,u32 * report)211 static void oa_report_id_clear(struct xe_oa_stream *stream, u32 *report)
212 {
213 	if (oa_report_header_64bit(stream))
214 		*(u64 *)report = 0;
215 	else
216 		*report = 0;
217 }
218 
oa_timestamp(struct xe_oa_stream * stream,void * report)219 static u64 oa_timestamp(struct xe_oa_stream *stream, void *report)
220 {
221 	return oa_report_header_64bit(stream) ?
222 		*((u64 *)report + 1) :
223 		*((u32 *)report + 1);
224 }
225 
oa_timestamp_clear(struct xe_oa_stream * stream,u32 * report)226 static void oa_timestamp_clear(struct xe_oa_stream *stream, u32 *report)
227 {
228 	if (oa_report_header_64bit(stream))
229 		*(u64 *)&report[2] = 0;
230 	else
231 		report[1] = 0;
232 }
233 
xe_oa_buffer_check_unlocked(struct xe_oa_stream * stream)234 static bool xe_oa_buffer_check_unlocked(struct xe_oa_stream *stream)
235 {
236 	u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);
237 	u32 tail, hw_tail, partial_report_size, available;
238 	int report_size = stream->oa_buffer.format->size;
239 	unsigned long flags;
240 
241 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
242 
243 	hw_tail = xe_oa_hw_tail_read(stream);
244 	hw_tail -= gtt_offset;
245 
246 	/*
247 	 * The tail pointer increases in 64 byte (cacheline size), not in report_size
248 	 * increments. Also report size may not be a power of 2. Compute potential
249 	 * partially landed report in OA buffer.
250 	 */
251 	partial_report_size = xe_oa_circ_diff(stream, hw_tail, stream->oa_buffer.tail);
252 	partial_report_size %= report_size;
253 
254 	/* Subtract partial amount off the tail */
255 	hw_tail = xe_oa_circ_diff(stream, hw_tail, partial_report_size);
256 
257 	tail = hw_tail;
258 
259 	/*
260 	 * Walk the stream backward until we find a report with report id and timestamp
261 	 * not 0. We can't tell whether a report has fully landed in memory before the
262 	 * report id and timestamp of the following report have landed.
263 	 *
264 	 * This is assuming that the writes of the OA unit land in memory in the order
265 	 * they were written.  If not : (╯°□°)╯︵ ┻━┻
266 	 */
267 	while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) {
268 		void *report = stream->oa_buffer.vaddr + tail;
269 
270 		if (oa_report_id(stream, report) || oa_timestamp(stream, report))
271 			break;
272 
273 		tail = xe_oa_circ_diff(stream, tail, report_size);
274 	}
275 
276 	if (xe_oa_circ_diff(stream, hw_tail, tail) > report_size)
277 		drm_dbg(&stream->oa->xe->drm,
278 			"unlanded report(s) head=0x%x tail=0x%x hw_tail=0x%x\n",
279 			stream->oa_buffer.head, tail, hw_tail);
280 
281 	stream->oa_buffer.tail = tail;
282 
283 	available = xe_oa_circ_diff(stream, stream->oa_buffer.tail, stream->oa_buffer.head);
284 	stream->pollin = available >= stream->wait_num_reports * report_size;
285 
286 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
287 
288 	return stream->pollin;
289 }
290 
xe_oa_poll_check_timer_cb(struct hrtimer * hrtimer)291 static enum hrtimer_restart xe_oa_poll_check_timer_cb(struct hrtimer *hrtimer)
292 {
293 	struct xe_oa_stream *stream =
294 		container_of(hrtimer, typeof(*stream), poll_check_timer);
295 
296 	if (xe_oa_buffer_check_unlocked(stream))
297 		wake_up(&stream->poll_wq);
298 
299 	hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period_ns));
300 
301 	return HRTIMER_RESTART;
302 }
303 
xe_oa_append_report(struct xe_oa_stream * stream,char __user * buf,size_t count,size_t * offset,const u8 * report)304 static int xe_oa_append_report(struct xe_oa_stream *stream, char __user *buf,
305 			       size_t count, size_t *offset, const u8 *report)
306 {
307 	int report_size = stream->oa_buffer.format->size;
308 	int report_size_partial;
309 	u8 *oa_buf_end;
310 
311 	if ((count - *offset) < report_size)
312 		return -ENOSPC;
313 
314 	buf += *offset;
315 
316 	oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size;
317 	report_size_partial = oa_buf_end - report;
318 
319 	if (report_size_partial < report_size) {
320 		if (copy_to_user(buf, report, report_size_partial))
321 			return -EFAULT;
322 		buf += report_size_partial;
323 
324 		if (copy_to_user(buf, stream->oa_buffer.vaddr,
325 				 report_size - report_size_partial))
326 			return -EFAULT;
327 	} else if (copy_to_user(buf, report, report_size)) {
328 		return -EFAULT;
329 	}
330 
331 	*offset += report_size;
332 
333 	return 0;
334 }
335 
xe_oa_append_reports(struct xe_oa_stream * stream,char __user * buf,size_t count,size_t * offset)336 static int xe_oa_append_reports(struct xe_oa_stream *stream, char __user *buf,
337 				size_t count, size_t *offset)
338 {
339 	int report_size = stream->oa_buffer.format->size;
340 	u8 *oa_buf_base = stream->oa_buffer.vaddr;
341 	u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);
342 	size_t start_offset = *offset;
343 	unsigned long flags;
344 	u32 head, tail;
345 	int ret = 0;
346 
347 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
348 	head = stream->oa_buffer.head;
349 	tail = stream->oa_buffer.tail;
350 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
351 
352 	xe_assert(stream->oa->xe,
353 		  head < stream->oa_buffer.circ_size && tail < stream->oa_buffer.circ_size);
354 
355 	for (; xe_oa_circ_diff(stream, tail, head);
356 	     head = xe_oa_circ_incr(stream, head, report_size)) {
357 		u8 *report = oa_buf_base + head;
358 
359 		ret = xe_oa_append_report(stream, buf, count, offset, report);
360 		if (ret)
361 			break;
362 
363 		if (!(stream->oa_buffer.circ_size % report_size)) {
364 			/* Clear out report id and timestamp to detect unlanded reports */
365 			oa_report_id_clear(stream, (void *)report);
366 			oa_timestamp_clear(stream, (void *)report);
367 		} else {
368 			u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size;
369 			u32 part = oa_buf_end - report;
370 
371 			/* Zero out the entire report */
372 			if (report_size <= part) {
373 				memset(report, 0, report_size);
374 			} else {
375 				memset(report, 0, part);
376 				memset(oa_buf_base, 0, report_size - part);
377 			}
378 		}
379 	}
380 
381 	if (start_offset != *offset) {
382 		struct xe_reg oaheadptr = __oa_regs(stream)->oa_head_ptr;
383 
384 		spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
385 		xe_mmio_write32(&stream->gt->mmio, oaheadptr,
386 				(head + gtt_offset) & OAG_OAHEADPTR_MASK);
387 		stream->oa_buffer.head = head;
388 		spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
389 	}
390 
391 	return ret;
392 }
393 
xe_oa_init_oa_buffer(struct xe_oa_stream * stream)394 static void xe_oa_init_oa_buffer(struct xe_oa_stream *stream)
395 {
396 	u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo);
397 	int size_exponent = __ffs(stream->oa_buffer.bo->size);
398 	u32 oa_buf = gtt_offset | OAG_OABUFFER_MEMORY_SELECT;
399 	struct xe_mmio *mmio = &stream->gt->mmio;
400 	unsigned long flags;
401 
402 	/*
403 	 * If oa buffer size is more than 16MB (exponent greater than 24), the
404 	 * oa buffer size field is multiplied by 8 in xe_oa_enable_metric_set.
405 	 */
406 	oa_buf |= REG_FIELD_PREP(OABUFFER_SIZE_MASK,
407 		size_exponent > 24 ? size_exponent - 20 : size_exponent - 17);
408 
409 	spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags);
410 
411 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_status, 0);
412 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_head_ptr,
413 			gtt_offset & OAG_OAHEADPTR_MASK);
414 	stream->oa_buffer.head = 0;
415 	/*
416 	 * PRM says: "This MMIO must be set before the OATAILPTR register and after the
417 	 * OAHEADPTR register. This is to enable proper functionality of the overflow bit".
418 	 */
419 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_buffer, oa_buf);
420 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_tail_ptr,
421 			gtt_offset & OAG_OATAILPTR_MASK);
422 
423 	/* Mark that we need updated tail pointer to read from */
424 	stream->oa_buffer.tail = 0;
425 
426 	spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags);
427 
428 	/* Zero out the OA buffer since we rely on zero report id and timestamp fields */
429 	memset(stream->oa_buffer.vaddr, 0, stream->oa_buffer.bo->size);
430 }
431 
__format_to_oactrl(const struct xe_oa_format * format,int counter_sel_mask)432 static u32 __format_to_oactrl(const struct xe_oa_format *format, int counter_sel_mask)
433 {
434 	return ((format->counter_select << (ffs(counter_sel_mask) - 1)) & counter_sel_mask) |
435 		REG_FIELD_PREP(OA_OACONTROL_REPORT_BC_MASK, format->bc_report) |
436 		REG_FIELD_PREP(OA_OACONTROL_COUNTER_SIZE_MASK, format->counter_size);
437 }
438 
__oa_ccs_select(struct xe_oa_stream * stream)439 static u32 __oa_ccs_select(struct xe_oa_stream *stream)
440 {
441 	u32 val;
442 
443 	if (stream->hwe->class != XE_ENGINE_CLASS_COMPUTE)
444 		return 0;
445 
446 	val = REG_FIELD_PREP(OAG_OACONTROL_OA_CCS_SELECT_MASK, stream->hwe->instance);
447 	xe_assert(stream->oa->xe,
448 		  REG_FIELD_GET(OAG_OACONTROL_OA_CCS_SELECT_MASK, val) == stream->hwe->instance);
449 	return val;
450 }
451 
__oactrl_used_bits(struct xe_oa_stream * stream)452 static u32 __oactrl_used_bits(struct xe_oa_stream *stream)
453 {
454 	return stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG ?
455 		OAG_OACONTROL_USED_BITS : OAM_OACONTROL_USED_BITS;
456 }
457 
xe_oa_enable(struct xe_oa_stream * stream)458 static void xe_oa_enable(struct xe_oa_stream *stream)
459 {
460 	const struct xe_oa_format *format = stream->oa_buffer.format;
461 	const struct xe_oa_regs *regs;
462 	u32 val;
463 
464 	/*
465 	 * BSpec: 46822: Bit 0. Even if stream->sample is 0, for OAR to function, the OA
466 	 * buffer must be correctly initialized
467 	 */
468 	xe_oa_init_oa_buffer(stream);
469 
470 	regs = __oa_regs(stream);
471 	val = __format_to_oactrl(format, regs->oa_ctrl_counter_select_mask) |
472 		__oa_ccs_select(stream) | OAG_OACONTROL_OA_COUNTER_ENABLE;
473 
474 	if (GRAPHICS_VER(stream->oa->xe) >= 20 &&
475 	    stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG)
476 		val |= OAG_OACONTROL_OA_PES_DISAG_EN;
477 
478 	xe_mmio_rmw32(&stream->gt->mmio, regs->oa_ctrl, __oactrl_used_bits(stream), val);
479 }
480 
xe_oa_disable(struct xe_oa_stream * stream)481 static void xe_oa_disable(struct xe_oa_stream *stream)
482 {
483 	struct xe_mmio *mmio = &stream->gt->mmio;
484 
485 	xe_mmio_rmw32(mmio, __oa_regs(stream)->oa_ctrl, __oactrl_used_bits(stream), 0);
486 	if (xe_mmio_wait32(mmio, __oa_regs(stream)->oa_ctrl,
487 			   OAG_OACONTROL_OA_COUNTER_ENABLE, 0, 50000, NULL, false))
488 		drm_err(&stream->oa->xe->drm,
489 			"wait for OA to be disabled timed out\n");
490 
491 	if (GRAPHICS_VERx100(stream->oa->xe) <= 1270 && GRAPHICS_VERx100(stream->oa->xe) != 1260) {
492 		/* <= XE_METEORLAKE except XE_PVC */
493 		xe_mmio_write32(mmio, OA_TLB_INV_CR, 1);
494 		if (xe_mmio_wait32(mmio, OA_TLB_INV_CR, 1, 0, 50000, NULL, false))
495 			drm_err(&stream->oa->xe->drm,
496 				"wait for OA tlb invalidate timed out\n");
497 	}
498 }
499 
xe_oa_wait_unlocked(struct xe_oa_stream * stream)500 static int xe_oa_wait_unlocked(struct xe_oa_stream *stream)
501 {
502 	/* We might wait indefinitely if periodic sampling is not enabled */
503 	if (!stream->periodic)
504 		return -EINVAL;
505 
506 	return wait_event_interruptible(stream->poll_wq,
507 					xe_oa_buffer_check_unlocked(stream));
508 }
509 
510 #define OASTATUS_RELEVANT_BITS (OASTATUS_MMIO_TRG_Q_FULL | OASTATUS_COUNTER_OVERFLOW | \
511 				OASTATUS_BUFFER_OVERFLOW | OASTATUS_REPORT_LOST)
512 
__xe_oa_read(struct xe_oa_stream * stream,char __user * buf,size_t count,size_t * offset)513 static int __xe_oa_read(struct xe_oa_stream *stream, char __user *buf,
514 			size_t count, size_t *offset)
515 {
516 	/* Only clear our bits to avoid side-effects */
517 	stream->oa_status = xe_mmio_rmw32(&stream->gt->mmio, __oa_regs(stream)->oa_status,
518 					  OASTATUS_RELEVANT_BITS, 0);
519 	/*
520 	 * Signal to userspace that there is non-zero OA status to read via
521 	 * @DRM_XE_OBSERVATION_IOCTL_STATUS observation stream fd ioctl
522 	 */
523 	if (stream->oa_status & OASTATUS_RELEVANT_BITS)
524 		return -EIO;
525 
526 	return xe_oa_append_reports(stream, buf, count, offset);
527 }
528 
xe_oa_read(struct file * file,char __user * buf,size_t count,loff_t * ppos)529 static ssize_t xe_oa_read(struct file *file, char __user *buf,
530 			  size_t count, loff_t *ppos)
531 {
532 	struct xe_oa_stream *stream = file->private_data;
533 	size_t offset = 0;
534 	int ret;
535 
536 	/* Can't read from disabled streams */
537 	if (!stream->enabled || !stream->sample)
538 		return -EINVAL;
539 
540 	if (!(file->f_flags & O_NONBLOCK)) {
541 		do {
542 			ret = xe_oa_wait_unlocked(stream);
543 			if (ret)
544 				return ret;
545 
546 			mutex_lock(&stream->stream_lock);
547 			ret = __xe_oa_read(stream, buf, count, &offset);
548 			mutex_unlock(&stream->stream_lock);
549 		} while (!offset && !ret);
550 	} else {
551 		mutex_lock(&stream->stream_lock);
552 		ret = __xe_oa_read(stream, buf, count, &offset);
553 		mutex_unlock(&stream->stream_lock);
554 	}
555 
556 	/*
557 	 * Typically we clear pollin here in order to wait for the new hrtimer callback
558 	 * before unblocking. The exception to this is if __xe_oa_read returns -ENOSPC,
559 	 * which means that more OA data is available than could fit in the user provided
560 	 * buffer. In this case we want the next poll() call to not block.
561 	 *
562 	 * Also in case of -EIO, we have already waited for data before returning
563 	 * -EIO, so need to wait again
564 	 */
565 	if (ret != -ENOSPC && ret != -EIO)
566 		stream->pollin = false;
567 
568 	/* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, -EINVAL, ... */
569 	return offset ?: (ret ?: -EAGAIN);
570 }
571 
xe_oa_poll_locked(struct xe_oa_stream * stream,struct file * file,poll_table * wait)572 static __poll_t xe_oa_poll_locked(struct xe_oa_stream *stream,
573 				  struct file *file, poll_table *wait)
574 {
575 	__poll_t events = 0;
576 
577 	poll_wait(file, &stream->poll_wq, wait);
578 
579 	/*
580 	 * We don't explicitly check whether there's something to read here since this
581 	 * path may be hot depending on what else userspace is polling, or on the timeout
582 	 * in use. We rely on hrtimer xe_oa_poll_check_timer_cb to notify us when there
583 	 * are samples to read
584 	 */
585 	if (stream->pollin)
586 		events |= EPOLLIN;
587 
588 	return events;
589 }
590 
xe_oa_poll(struct file * file,poll_table * wait)591 static __poll_t xe_oa_poll(struct file *file, poll_table *wait)
592 {
593 	struct xe_oa_stream *stream = file->private_data;
594 	__poll_t ret;
595 
596 	mutex_lock(&stream->stream_lock);
597 	ret = xe_oa_poll_locked(stream, file, wait);
598 	mutex_unlock(&stream->stream_lock);
599 
600 	return ret;
601 }
602 
xe_oa_lock_vma(struct xe_exec_queue * q)603 static void xe_oa_lock_vma(struct xe_exec_queue *q)
604 {
605 	if (q->vm) {
606 		down_read(&q->vm->lock);
607 		xe_vm_lock(q->vm, false);
608 	}
609 }
610 
xe_oa_unlock_vma(struct xe_exec_queue * q)611 static void xe_oa_unlock_vma(struct xe_exec_queue *q)
612 {
613 	if (q->vm) {
614 		xe_vm_unlock(q->vm);
615 		up_read(&q->vm->lock);
616 	}
617 }
618 
xe_oa_submit_bb(struct xe_oa_stream * stream,enum xe_oa_submit_deps deps,struct xe_bb * bb)619 static struct dma_fence *xe_oa_submit_bb(struct xe_oa_stream *stream, enum xe_oa_submit_deps deps,
620 					 struct xe_bb *bb)
621 {
622 	struct xe_exec_queue *q = stream->exec_q ?: stream->k_exec_q;
623 	struct xe_sched_job *job;
624 	struct dma_fence *fence;
625 	int err = 0;
626 
627 	xe_oa_lock_vma(q);
628 
629 	job = xe_bb_create_job(q, bb);
630 	if (IS_ERR(job)) {
631 		err = PTR_ERR(job);
632 		goto exit;
633 	}
634 	job->ggtt = true;
635 
636 	if (deps == XE_OA_SUBMIT_ADD_DEPS) {
637 		for (int i = 0; i < stream->num_syncs && !err; i++)
638 			err = xe_sync_entry_add_deps(&stream->syncs[i], job);
639 		if (err) {
640 			drm_dbg(&stream->oa->xe->drm, "xe_sync_entry_add_deps err %d\n", err);
641 			goto err_put_job;
642 		}
643 	}
644 
645 	xe_sched_job_arm(job);
646 	fence = dma_fence_get(&job->drm.s_fence->finished);
647 	xe_sched_job_push(job);
648 
649 	xe_oa_unlock_vma(q);
650 
651 	return fence;
652 err_put_job:
653 	xe_sched_job_put(job);
654 exit:
655 	xe_oa_unlock_vma(q);
656 	return ERR_PTR(err);
657 }
658 
write_cs_mi_lri(struct xe_bb * bb,const struct xe_oa_reg * reg_data,u32 n_regs)659 static void write_cs_mi_lri(struct xe_bb *bb, const struct xe_oa_reg *reg_data, u32 n_regs)
660 {
661 	u32 i;
662 
663 #define MI_LOAD_REGISTER_IMM_MAX_REGS (126)
664 
665 	for (i = 0; i < n_regs; i++) {
666 		if ((i % MI_LOAD_REGISTER_IMM_MAX_REGS) == 0) {
667 			u32 n_lri = min_t(u32, n_regs - i,
668 					  MI_LOAD_REGISTER_IMM_MAX_REGS);
669 
670 			bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(n_lri);
671 		}
672 		bb->cs[bb->len++] = reg_data[i].addr.addr;
673 		bb->cs[bb->len++] = reg_data[i].value;
674 	}
675 }
676 
num_lri_dwords(int num_regs)677 static int num_lri_dwords(int num_regs)
678 {
679 	int count = 0;
680 
681 	if (num_regs > 0) {
682 		count += DIV_ROUND_UP(num_regs, MI_LOAD_REGISTER_IMM_MAX_REGS);
683 		count += num_regs * 2;
684 	}
685 
686 	return count;
687 }
688 
xe_oa_free_oa_buffer(struct xe_oa_stream * stream)689 static void xe_oa_free_oa_buffer(struct xe_oa_stream *stream)
690 {
691 	xe_bo_unpin_map_no_vm(stream->oa_buffer.bo);
692 }
693 
xe_oa_free_configs(struct xe_oa_stream * stream)694 static void xe_oa_free_configs(struct xe_oa_stream *stream)
695 {
696 	struct xe_oa_config_bo *oa_bo, *tmp;
697 
698 	xe_oa_config_put(stream->oa_config);
699 	llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node)
700 		free_oa_config_bo(oa_bo, stream->last_fence);
701 	dma_fence_put(stream->last_fence);
702 }
703 
xe_oa_load_with_lri(struct xe_oa_stream * stream,struct xe_oa_reg * reg_lri,u32 count)704 static int xe_oa_load_with_lri(struct xe_oa_stream *stream, struct xe_oa_reg *reg_lri, u32 count)
705 {
706 	struct dma_fence *fence;
707 	struct xe_bb *bb;
708 	int err;
709 
710 	bb = xe_bb_new(stream->gt, 2 * count + 1, false);
711 	if (IS_ERR(bb)) {
712 		err = PTR_ERR(bb);
713 		goto exit;
714 	}
715 
716 	write_cs_mi_lri(bb, reg_lri, count);
717 
718 	fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_NO_DEPS, bb);
719 	if (IS_ERR(fence)) {
720 		err = PTR_ERR(fence);
721 		goto free_bb;
722 	}
723 	xe_bb_free(bb, fence);
724 	dma_fence_put(fence);
725 
726 	return 0;
727 free_bb:
728 	xe_bb_free(bb, NULL);
729 exit:
730 	return err;
731 }
732 
xe_oa_configure_oar_context(struct xe_oa_stream * stream,bool enable)733 static int xe_oa_configure_oar_context(struct xe_oa_stream *stream, bool enable)
734 {
735 	const struct xe_oa_format *format = stream->oa_buffer.format;
736 	u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) |
737 		(enable ? OAR_OACONTROL_COUNTER_ENABLE : 0);
738 
739 	struct xe_oa_reg reg_lri[] = {
740 		{
741 			OACTXCONTROL(stream->hwe->mmio_base),
742 			enable ? OA_COUNTER_RESUME : 0,
743 		},
744 		{
745 			OAR_OACONTROL,
746 			oacontrol,
747 		},
748 		{
749 			RING_CONTEXT_CONTROL(stream->hwe->mmio_base),
750 			_MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE,
751 				      enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0)
752 		},
753 	};
754 
755 	return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri));
756 }
757 
xe_oa_configure_oac_context(struct xe_oa_stream * stream,bool enable)758 static int xe_oa_configure_oac_context(struct xe_oa_stream *stream, bool enable)
759 {
760 	const struct xe_oa_format *format = stream->oa_buffer.format;
761 	u32 oacontrol = __format_to_oactrl(format, OAR_OACONTROL_COUNTER_SEL_MASK) |
762 		(enable ? OAR_OACONTROL_COUNTER_ENABLE : 0);
763 	struct xe_oa_reg reg_lri[] = {
764 		{
765 			OACTXCONTROL(stream->hwe->mmio_base),
766 			enable ? OA_COUNTER_RESUME : 0,
767 		},
768 		{
769 			OAC_OACONTROL,
770 			oacontrol
771 		},
772 		{
773 			RING_CONTEXT_CONTROL(stream->hwe->mmio_base),
774 			_MASKED_FIELD(CTX_CTRL_OAC_CONTEXT_ENABLE,
775 				      enable ? CTX_CTRL_OAC_CONTEXT_ENABLE : 0) |
776 			_MASKED_FIELD(CTX_CTRL_RUN_ALONE, enable ? CTX_CTRL_RUN_ALONE : 0),
777 		},
778 	};
779 
780 	/* Set ccs select to enable programming of OAC_OACONTROL */
781 	xe_mmio_write32(&stream->gt->mmio, __oa_regs(stream)->oa_ctrl,
782 			__oa_ccs_select(stream));
783 
784 	return xe_oa_load_with_lri(stream, reg_lri, ARRAY_SIZE(reg_lri));
785 }
786 
xe_oa_configure_oa_context(struct xe_oa_stream * stream,bool enable)787 static int xe_oa_configure_oa_context(struct xe_oa_stream *stream, bool enable)
788 {
789 	switch (stream->hwe->class) {
790 	case XE_ENGINE_CLASS_RENDER:
791 		return xe_oa_configure_oar_context(stream, enable);
792 	case XE_ENGINE_CLASS_COMPUTE:
793 		return xe_oa_configure_oac_context(stream, enable);
794 	default:
795 		/* Video engines do not support MI_REPORT_PERF_COUNT */
796 		return 0;
797 	}
798 }
799 
800 #define HAS_OA_BPC_REPORTING(xe) (GRAPHICS_VERx100(xe) >= 1255)
801 
oag_configure_mmio_trigger(const struct xe_oa_stream * stream,bool enable)802 static u32 oag_configure_mmio_trigger(const struct xe_oa_stream *stream, bool enable)
803 {
804 	return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_MMIO_TRG,
805 			     enable && stream && stream->sample ?
806 			     0 : OAG_OA_DEBUG_DISABLE_MMIO_TRG);
807 }
808 
xe_oa_disable_metric_set(struct xe_oa_stream * stream)809 static void xe_oa_disable_metric_set(struct xe_oa_stream *stream)
810 {
811 	struct xe_mmio *mmio = &stream->gt->mmio;
812 	u32 sqcnt1;
813 
814 	/*
815 	 * Wa_1508761755:xehpsdv, dg2
816 	 * Enable thread stall DOP gating and EU DOP gating.
817 	 */
818 	if (stream->oa->xe->info.platform == XE_DG2) {
819 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN,
820 					  _MASKED_BIT_DISABLE(STALL_DOP_GATING_DISABLE));
821 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2,
822 					  _MASKED_BIT_DISABLE(DISABLE_DOP_GATING));
823 	}
824 
825 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug,
826 			oag_configure_mmio_trigger(stream, false));
827 
828 	/* disable the context save/restore or OAR counters */
829 	if (stream->exec_q)
830 		xe_oa_configure_oa_context(stream, false);
831 
832 	/* Make sure we disable noa to save power. */
833 	xe_mmio_rmw32(mmio, RPM_CONFIG1, GT_NOA_ENABLE, 0);
834 
835 	sqcnt1 = SQCNT1_PMON_ENABLE |
836 		 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0);
837 
838 	/* Reset PMON Enable to save power. */
839 	xe_mmio_rmw32(mmio, XELPMP_SQCNT1, sqcnt1, 0);
840 }
841 
xe_oa_stream_destroy(struct xe_oa_stream * stream)842 static void xe_oa_stream_destroy(struct xe_oa_stream *stream)
843 {
844 	struct xe_oa_unit *u = stream->hwe->oa_unit;
845 	struct xe_gt *gt = stream->hwe->gt;
846 
847 	if (WARN_ON(stream != u->exclusive_stream))
848 		return;
849 
850 	WRITE_ONCE(u->exclusive_stream, NULL);
851 
852 	mutex_destroy(&stream->stream_lock);
853 
854 	xe_oa_disable_metric_set(stream);
855 	xe_exec_queue_put(stream->k_exec_q);
856 
857 	xe_oa_free_oa_buffer(stream);
858 
859 	xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL);
860 	xe_pm_runtime_put(stream->oa->xe);
861 
862 	/* Wa_1509372804:pvc: Unset the override of GUCRC mode to enable rc6 */
863 	if (stream->override_gucrc)
864 		xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(&gt->uc.guc.pc));
865 
866 	xe_oa_free_configs(stream);
867 	xe_file_put(stream->xef);
868 }
869 
xe_oa_alloc_oa_buffer(struct xe_oa_stream * stream,size_t size)870 static int xe_oa_alloc_oa_buffer(struct xe_oa_stream *stream, size_t size)
871 {
872 	struct xe_bo *bo;
873 
874 	bo = xe_bo_create_pin_map(stream->oa->xe, stream->gt->tile, NULL,
875 				  size, ttm_bo_type_kernel,
876 				  XE_BO_FLAG_SYSTEM | XE_BO_FLAG_GGTT);
877 	if (IS_ERR(bo))
878 		return PTR_ERR(bo);
879 
880 	stream->oa_buffer.bo = bo;
881 	/* mmap implementation requires OA buffer to be in system memory */
882 	xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0);
883 	stream->oa_buffer.vaddr = bo->vmap.vaddr;
884 	return 0;
885 }
886 
887 static struct xe_oa_config_bo *
__xe_oa_alloc_config_buffer(struct xe_oa_stream * stream,struct xe_oa_config * oa_config)888 __xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config)
889 {
890 	struct xe_oa_config_bo *oa_bo;
891 	size_t config_length;
892 	struct xe_bb *bb;
893 
894 	oa_bo = kzalloc(sizeof(*oa_bo), GFP_KERNEL);
895 	if (!oa_bo)
896 		return ERR_PTR(-ENOMEM);
897 
898 	config_length = num_lri_dwords(oa_config->regs_len);
899 	config_length = ALIGN(sizeof(u32) * config_length, XE_PAGE_SIZE) / sizeof(u32);
900 
901 	bb = xe_bb_new(stream->gt, config_length, false);
902 	if (IS_ERR(bb))
903 		goto err_free;
904 
905 	write_cs_mi_lri(bb, oa_config->regs, oa_config->regs_len);
906 
907 	oa_bo->bb = bb;
908 	oa_bo->oa_config = xe_oa_config_get(oa_config);
909 	llist_add(&oa_bo->node, &stream->oa_config_bos);
910 
911 	return oa_bo;
912 err_free:
913 	kfree(oa_bo);
914 	return ERR_CAST(bb);
915 }
916 
917 static struct xe_oa_config_bo *
xe_oa_alloc_config_buffer(struct xe_oa_stream * stream,struct xe_oa_config * oa_config)918 xe_oa_alloc_config_buffer(struct xe_oa_stream *stream, struct xe_oa_config *oa_config)
919 {
920 	struct xe_oa_config_bo *oa_bo;
921 
922 	/* Look for the buffer in the already allocated BOs attached to the stream */
923 	llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) {
924 		if (oa_bo->oa_config == oa_config &&
925 		    memcmp(oa_bo->oa_config->uuid, oa_config->uuid,
926 			   sizeof(oa_config->uuid)) == 0)
927 			goto out;
928 	}
929 
930 	oa_bo = __xe_oa_alloc_config_buffer(stream, oa_config);
931 out:
932 	return oa_bo;
933 }
934 
xe_oa_update_last_fence(struct xe_oa_stream * stream,struct dma_fence * fence)935 static void xe_oa_update_last_fence(struct xe_oa_stream *stream, struct dma_fence *fence)
936 {
937 	dma_fence_put(stream->last_fence);
938 	stream->last_fence = dma_fence_get(fence);
939 }
940 
xe_oa_fence_work_fn(struct work_struct * w)941 static void xe_oa_fence_work_fn(struct work_struct *w)
942 {
943 	struct xe_oa_fence *ofence = container_of(w, typeof(*ofence), work.work);
944 
945 	/* Signal fence to indicate new OA configuration is active */
946 	dma_fence_signal(&ofence->base);
947 	dma_fence_put(&ofence->base);
948 }
949 
xe_oa_config_cb(struct dma_fence * fence,struct dma_fence_cb * cb)950 static void xe_oa_config_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
951 {
952 	/* Additional empirical delay needed for NOA programming after registers are written */
953 #define NOA_PROGRAM_ADDITIONAL_DELAY_US 500
954 
955 	struct xe_oa_fence *ofence = container_of(cb, typeof(*ofence), cb);
956 
957 	INIT_DELAYED_WORK(&ofence->work, xe_oa_fence_work_fn);
958 	queue_delayed_work(system_unbound_wq, &ofence->work,
959 			   usecs_to_jiffies(NOA_PROGRAM_ADDITIONAL_DELAY_US));
960 	dma_fence_put(fence);
961 }
962 
xe_oa_get_driver_name(struct dma_fence * fence)963 static const char *xe_oa_get_driver_name(struct dma_fence *fence)
964 {
965 	return "xe_oa";
966 }
967 
xe_oa_get_timeline_name(struct dma_fence * fence)968 static const char *xe_oa_get_timeline_name(struct dma_fence *fence)
969 {
970 	return "unbound";
971 }
972 
973 static const struct dma_fence_ops xe_oa_fence_ops = {
974 	.get_driver_name = xe_oa_get_driver_name,
975 	.get_timeline_name = xe_oa_get_timeline_name,
976 };
977 
xe_oa_emit_oa_config(struct xe_oa_stream * stream,struct xe_oa_config * config)978 static int xe_oa_emit_oa_config(struct xe_oa_stream *stream, struct xe_oa_config *config)
979 {
980 #define NOA_PROGRAM_ADDITIONAL_DELAY_US 500
981 	struct xe_oa_config_bo *oa_bo;
982 	struct xe_oa_fence *ofence;
983 	int i, err, num_signal = 0;
984 	struct dma_fence *fence;
985 
986 	ofence = kzalloc(sizeof(*ofence), GFP_KERNEL);
987 	if (!ofence) {
988 		err = -ENOMEM;
989 		goto exit;
990 	}
991 
992 	oa_bo = xe_oa_alloc_config_buffer(stream, config);
993 	if (IS_ERR(oa_bo)) {
994 		err = PTR_ERR(oa_bo);
995 		goto exit;
996 	}
997 
998 	/* Emit OA configuration batch */
999 	fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_ADD_DEPS, oa_bo->bb);
1000 	if (IS_ERR(fence)) {
1001 		err = PTR_ERR(fence);
1002 		goto exit;
1003 	}
1004 
1005 	/* Point of no return: initialize and set fence to signal */
1006 	spin_lock_init(&ofence->lock);
1007 	dma_fence_init(&ofence->base, &xe_oa_fence_ops, &ofence->lock, 0, 0);
1008 
1009 	for (i = 0; i < stream->num_syncs; i++) {
1010 		if (stream->syncs[i].flags & DRM_XE_SYNC_FLAG_SIGNAL)
1011 			num_signal++;
1012 		xe_sync_entry_signal(&stream->syncs[i], &ofence->base);
1013 	}
1014 
1015 	/* Additional dma_fence_get in case we dma_fence_wait */
1016 	if (!num_signal)
1017 		dma_fence_get(&ofence->base);
1018 
1019 	/* Update last fence too before adding callback */
1020 	xe_oa_update_last_fence(stream, fence);
1021 
1022 	/* Add job fence callback to schedule work to signal ofence->base */
1023 	err = dma_fence_add_callback(fence, &ofence->cb, xe_oa_config_cb);
1024 	xe_gt_assert(stream->gt, !err || err == -ENOENT);
1025 	if (err == -ENOENT)
1026 		xe_oa_config_cb(fence, &ofence->cb);
1027 
1028 	/* If nothing needs to be signaled we wait synchronously */
1029 	if (!num_signal) {
1030 		dma_fence_wait(&ofence->base, false);
1031 		dma_fence_put(&ofence->base);
1032 	}
1033 
1034 	/* Done with syncs */
1035 	for (i = 0; i < stream->num_syncs; i++)
1036 		xe_sync_entry_cleanup(&stream->syncs[i]);
1037 	kfree(stream->syncs);
1038 
1039 	return 0;
1040 exit:
1041 	kfree(ofence);
1042 	return err;
1043 }
1044 
oag_report_ctx_switches(const struct xe_oa_stream * stream)1045 static u32 oag_report_ctx_switches(const struct xe_oa_stream *stream)
1046 {
1047 	/* If user didn't require OA reports, ask HW not to emit ctx switch reports */
1048 	return _MASKED_FIELD(OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS,
1049 			     stream->sample ?
1050 			     0 : OAG_OA_DEBUG_DISABLE_CTX_SWITCH_REPORTS);
1051 }
1052 
oag_buf_size_select(const struct xe_oa_stream * stream)1053 static u32 oag_buf_size_select(const struct xe_oa_stream *stream)
1054 {
1055 	return _MASKED_FIELD(OAG_OA_DEBUG_BUF_SIZE_SELECT,
1056 			     stream->oa_buffer.bo->size > SZ_16M ?
1057 			     OAG_OA_DEBUG_BUF_SIZE_SELECT : 0);
1058 }
1059 
xe_oa_enable_metric_set(struct xe_oa_stream * stream)1060 static int xe_oa_enable_metric_set(struct xe_oa_stream *stream)
1061 {
1062 	struct xe_mmio *mmio = &stream->gt->mmio;
1063 	u32 oa_debug, sqcnt1;
1064 	int ret;
1065 
1066 	/*
1067 	 * Wa_1508761755:xehpsdv, dg2
1068 	 * EU NOA signals behave incorrectly if EU clock gating is enabled.
1069 	 * Disable thread stall DOP gating and EU DOP gating.
1070 	 */
1071 	if (stream->oa->xe->info.platform == XE_DG2) {
1072 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN,
1073 					  _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
1074 		xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2,
1075 					  _MASKED_BIT_ENABLE(DISABLE_DOP_GATING));
1076 	}
1077 
1078 	/* Disable clk ratio reports */
1079 	oa_debug = OAG_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
1080 		OAG_OA_DEBUG_INCLUDE_CLK_RATIO;
1081 
1082 	if (GRAPHICS_VER(stream->oa->xe) >= 20)
1083 		oa_debug |=
1084 			/* The three bits below are needed to get PEC counters running */
1085 			OAG_OA_DEBUG_START_TRIGGER_SCOPE_CONTROL |
1086 			OAG_OA_DEBUG_DISABLE_START_TRG_2_COUNT_QUAL |
1087 			OAG_OA_DEBUG_DISABLE_START_TRG_1_COUNT_QUAL;
1088 
1089 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug,
1090 			_MASKED_BIT_ENABLE(oa_debug) |
1091 			oag_report_ctx_switches(stream) |
1092 			oag_buf_size_select(stream) |
1093 			oag_configure_mmio_trigger(stream, true));
1094 
1095 	xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ?
1096 			(OAG_OAGLBCTXCTRL_COUNTER_RESUME |
1097 			 OAG_OAGLBCTXCTRL_TIMER_ENABLE |
1098 			 REG_FIELD_PREP(OAG_OAGLBCTXCTRL_TIMER_PERIOD_MASK,
1099 					stream->period_exponent)) : 0);
1100 
1101 	/*
1102 	 * Initialize Super Queue Internal Cnt Register
1103 	 * Set PMON Enable in order to collect valid metrics
1104 	 * Enable bytes per clock reporting
1105 	 */
1106 	sqcnt1 = SQCNT1_PMON_ENABLE |
1107 		 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0);
1108 
1109 	xe_mmio_rmw32(mmio, XELPMP_SQCNT1, 0, sqcnt1);
1110 
1111 	/* Configure OAR/OAC */
1112 	if (stream->exec_q) {
1113 		ret = xe_oa_configure_oa_context(stream, true);
1114 		if (ret)
1115 			return ret;
1116 	}
1117 
1118 	return xe_oa_emit_oa_config(stream, stream->oa_config);
1119 }
1120 
decode_oa_format(struct xe_oa * oa,u64 fmt,enum xe_oa_format_name * name)1121 static int decode_oa_format(struct xe_oa *oa, u64 fmt, enum xe_oa_format_name *name)
1122 {
1123 	u32 counter_size = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SIZE, fmt);
1124 	u32 counter_sel = FIELD_GET(DRM_XE_OA_FORMAT_MASK_COUNTER_SEL, fmt);
1125 	u32 bc_report = FIELD_GET(DRM_XE_OA_FORMAT_MASK_BC_REPORT, fmt);
1126 	u32 type = FIELD_GET(DRM_XE_OA_FORMAT_MASK_FMT_TYPE, fmt);
1127 	int idx;
1128 
1129 	for_each_set_bit(idx, oa->format_mask, __XE_OA_FORMAT_MAX) {
1130 		const struct xe_oa_format *f = &oa->oa_formats[idx];
1131 
1132 		if (counter_size == f->counter_size && bc_report == f->bc_report &&
1133 		    type == f->type && counter_sel == f->counter_select) {
1134 			*name = idx;
1135 			return 0;
1136 		}
1137 	}
1138 
1139 	return -EINVAL;
1140 }
1141 
xe_oa_set_prop_oa_unit_id(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1142 static int xe_oa_set_prop_oa_unit_id(struct xe_oa *oa, u64 value,
1143 				     struct xe_oa_open_param *param)
1144 {
1145 	if (value >= oa->oa_unit_ids) {
1146 		drm_dbg(&oa->xe->drm, "OA unit ID out of range %lld\n", value);
1147 		return -EINVAL;
1148 	}
1149 	param->oa_unit_id = value;
1150 	return 0;
1151 }
1152 
xe_oa_set_prop_sample_oa(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1153 static int xe_oa_set_prop_sample_oa(struct xe_oa *oa, u64 value,
1154 				    struct xe_oa_open_param *param)
1155 {
1156 	param->sample = value;
1157 	return 0;
1158 }
1159 
xe_oa_set_prop_metric_set(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1160 static int xe_oa_set_prop_metric_set(struct xe_oa *oa, u64 value,
1161 				     struct xe_oa_open_param *param)
1162 {
1163 	param->metric_set = value;
1164 	return 0;
1165 }
1166 
xe_oa_set_prop_oa_format(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1167 static int xe_oa_set_prop_oa_format(struct xe_oa *oa, u64 value,
1168 				    struct xe_oa_open_param *param)
1169 {
1170 	int ret = decode_oa_format(oa, value, &param->oa_format);
1171 
1172 	if (ret) {
1173 		drm_dbg(&oa->xe->drm, "Unsupported OA report format %#llx\n", value);
1174 		return ret;
1175 	}
1176 	return 0;
1177 }
1178 
xe_oa_set_prop_oa_exponent(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1179 static int xe_oa_set_prop_oa_exponent(struct xe_oa *oa, u64 value,
1180 				      struct xe_oa_open_param *param)
1181 {
1182 #define OA_EXPONENT_MAX 31
1183 
1184 	if (value > OA_EXPONENT_MAX) {
1185 		drm_dbg(&oa->xe->drm, "OA timer exponent too high (> %u)\n", OA_EXPONENT_MAX);
1186 		return -EINVAL;
1187 	}
1188 	param->period_exponent = value;
1189 	return 0;
1190 }
1191 
xe_oa_set_prop_disabled(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1192 static int xe_oa_set_prop_disabled(struct xe_oa *oa, u64 value,
1193 				   struct xe_oa_open_param *param)
1194 {
1195 	param->disabled = value;
1196 	return 0;
1197 }
1198 
xe_oa_set_prop_exec_queue_id(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1199 static int xe_oa_set_prop_exec_queue_id(struct xe_oa *oa, u64 value,
1200 					struct xe_oa_open_param *param)
1201 {
1202 	param->exec_queue_id = value;
1203 	return 0;
1204 }
1205 
xe_oa_set_prop_engine_instance(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1206 static int xe_oa_set_prop_engine_instance(struct xe_oa *oa, u64 value,
1207 					  struct xe_oa_open_param *param)
1208 {
1209 	param->engine_instance = value;
1210 	return 0;
1211 }
1212 
xe_oa_set_no_preempt(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1213 static int xe_oa_set_no_preempt(struct xe_oa *oa, u64 value,
1214 				struct xe_oa_open_param *param)
1215 {
1216 	param->no_preempt = value;
1217 	return 0;
1218 }
1219 
xe_oa_set_prop_num_syncs(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1220 static int xe_oa_set_prop_num_syncs(struct xe_oa *oa, u64 value,
1221 				    struct xe_oa_open_param *param)
1222 {
1223 	param->num_syncs = value;
1224 	return 0;
1225 }
1226 
xe_oa_set_prop_syncs_user(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1227 static int xe_oa_set_prop_syncs_user(struct xe_oa *oa, u64 value,
1228 				     struct xe_oa_open_param *param)
1229 {
1230 	param->syncs_user = u64_to_user_ptr(value);
1231 	return 0;
1232 }
1233 
xe_oa_set_prop_oa_buffer_size(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1234 static int xe_oa_set_prop_oa_buffer_size(struct xe_oa *oa, u64 value,
1235 					 struct xe_oa_open_param *param)
1236 {
1237 	if (!is_power_of_2(value) || value < SZ_128K || value > SZ_128M) {
1238 		drm_dbg(&oa->xe->drm, "OA buffer size invalid %llu\n", value);
1239 		return -EINVAL;
1240 	}
1241 	param->oa_buffer_size = value;
1242 	return 0;
1243 }
1244 
xe_oa_set_prop_wait_num_reports(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1245 static int xe_oa_set_prop_wait_num_reports(struct xe_oa *oa, u64 value,
1246 					   struct xe_oa_open_param *param)
1247 {
1248 	if (!value) {
1249 		drm_dbg(&oa->xe->drm, "wait_num_reports %llu\n", value);
1250 		return -EINVAL;
1251 	}
1252 	param->wait_num_reports = value;
1253 	return 0;
1254 }
1255 
xe_oa_set_prop_ret_inval(struct xe_oa * oa,u64 value,struct xe_oa_open_param * param)1256 static int xe_oa_set_prop_ret_inval(struct xe_oa *oa, u64 value,
1257 				    struct xe_oa_open_param *param)
1258 {
1259 	return -EINVAL;
1260 }
1261 
1262 typedef int (*xe_oa_set_property_fn)(struct xe_oa *oa, u64 value,
1263 				     struct xe_oa_open_param *param);
1264 static const xe_oa_set_property_fn xe_oa_set_property_funcs_open[] = {
1265 	[DRM_XE_OA_PROPERTY_OA_UNIT_ID] = xe_oa_set_prop_oa_unit_id,
1266 	[DRM_XE_OA_PROPERTY_SAMPLE_OA] = xe_oa_set_prop_sample_oa,
1267 	[DRM_XE_OA_PROPERTY_OA_METRIC_SET] = xe_oa_set_prop_metric_set,
1268 	[DRM_XE_OA_PROPERTY_OA_FORMAT] = xe_oa_set_prop_oa_format,
1269 	[DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT] = xe_oa_set_prop_oa_exponent,
1270 	[DRM_XE_OA_PROPERTY_OA_DISABLED] = xe_oa_set_prop_disabled,
1271 	[DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID] = xe_oa_set_prop_exec_queue_id,
1272 	[DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE] = xe_oa_set_prop_engine_instance,
1273 	[DRM_XE_OA_PROPERTY_NO_PREEMPT] = xe_oa_set_no_preempt,
1274 	[DRM_XE_OA_PROPERTY_NUM_SYNCS] = xe_oa_set_prop_num_syncs,
1275 	[DRM_XE_OA_PROPERTY_SYNCS] = xe_oa_set_prop_syncs_user,
1276 	[DRM_XE_OA_PROPERTY_OA_BUFFER_SIZE] = xe_oa_set_prop_oa_buffer_size,
1277 	[DRM_XE_OA_PROPERTY_WAIT_NUM_REPORTS] = xe_oa_set_prop_wait_num_reports,
1278 };
1279 
1280 static const xe_oa_set_property_fn xe_oa_set_property_funcs_config[] = {
1281 	[DRM_XE_OA_PROPERTY_OA_UNIT_ID] = xe_oa_set_prop_ret_inval,
1282 	[DRM_XE_OA_PROPERTY_SAMPLE_OA] = xe_oa_set_prop_ret_inval,
1283 	[DRM_XE_OA_PROPERTY_OA_METRIC_SET] = xe_oa_set_prop_metric_set,
1284 	[DRM_XE_OA_PROPERTY_OA_FORMAT] = xe_oa_set_prop_ret_inval,
1285 	[DRM_XE_OA_PROPERTY_OA_PERIOD_EXPONENT] = xe_oa_set_prop_ret_inval,
1286 	[DRM_XE_OA_PROPERTY_OA_DISABLED] = xe_oa_set_prop_ret_inval,
1287 	[DRM_XE_OA_PROPERTY_EXEC_QUEUE_ID] = xe_oa_set_prop_ret_inval,
1288 	[DRM_XE_OA_PROPERTY_OA_ENGINE_INSTANCE] = xe_oa_set_prop_ret_inval,
1289 	[DRM_XE_OA_PROPERTY_NO_PREEMPT] = xe_oa_set_prop_ret_inval,
1290 	[DRM_XE_OA_PROPERTY_NUM_SYNCS] = xe_oa_set_prop_num_syncs,
1291 	[DRM_XE_OA_PROPERTY_SYNCS] = xe_oa_set_prop_syncs_user,
1292 	[DRM_XE_OA_PROPERTY_OA_BUFFER_SIZE] = xe_oa_set_prop_ret_inval,
1293 	[DRM_XE_OA_PROPERTY_WAIT_NUM_REPORTS] = xe_oa_set_prop_ret_inval,
1294 };
1295 
xe_oa_user_ext_set_property(struct xe_oa * oa,enum xe_oa_user_extn_from from,u64 extension,struct xe_oa_open_param * param)1296 static int xe_oa_user_ext_set_property(struct xe_oa *oa, enum xe_oa_user_extn_from from,
1297 				       u64 extension, struct xe_oa_open_param *param)
1298 {
1299 	u64 __user *address = u64_to_user_ptr(extension);
1300 	struct drm_xe_ext_set_property ext;
1301 	int err;
1302 	u32 idx;
1303 
1304 	err = __copy_from_user(&ext, address, sizeof(ext));
1305 	if (XE_IOCTL_DBG(oa->xe, err))
1306 		return -EFAULT;
1307 
1308 	BUILD_BUG_ON(ARRAY_SIZE(xe_oa_set_property_funcs_open) !=
1309 		     ARRAY_SIZE(xe_oa_set_property_funcs_config));
1310 
1311 	if (XE_IOCTL_DBG(oa->xe, ext.property >= ARRAY_SIZE(xe_oa_set_property_funcs_open)) ||
1312 	    XE_IOCTL_DBG(oa->xe, ext.pad))
1313 		return -EINVAL;
1314 
1315 	idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_oa_set_property_funcs_open));
1316 
1317 	if (from == XE_OA_USER_EXTN_FROM_CONFIG)
1318 		return xe_oa_set_property_funcs_config[idx](oa, ext.value, param);
1319 	else
1320 		return xe_oa_set_property_funcs_open[idx](oa, ext.value, param);
1321 }
1322 
1323 typedef int (*xe_oa_user_extension_fn)(struct xe_oa *oa,  enum xe_oa_user_extn_from from,
1324 				       u64 extension, struct xe_oa_open_param *param);
1325 static const xe_oa_user_extension_fn xe_oa_user_extension_funcs[] = {
1326 	[DRM_XE_OA_EXTENSION_SET_PROPERTY] = xe_oa_user_ext_set_property,
1327 };
1328 
1329 #define MAX_USER_EXTENSIONS	16
xe_oa_user_extensions(struct xe_oa * oa,enum xe_oa_user_extn_from from,u64 extension,int ext_number,struct xe_oa_open_param * param)1330 static int xe_oa_user_extensions(struct xe_oa *oa, enum xe_oa_user_extn_from from, u64 extension,
1331 				 int ext_number, struct xe_oa_open_param *param)
1332 {
1333 	u64 __user *address = u64_to_user_ptr(extension);
1334 	struct drm_xe_user_extension ext;
1335 	int err;
1336 	u32 idx;
1337 
1338 	if (XE_IOCTL_DBG(oa->xe, ext_number >= MAX_USER_EXTENSIONS))
1339 		return -E2BIG;
1340 
1341 	err = __copy_from_user(&ext, address, sizeof(ext));
1342 	if (XE_IOCTL_DBG(oa->xe, err))
1343 		return -EFAULT;
1344 
1345 	if (XE_IOCTL_DBG(oa->xe, ext.pad) ||
1346 	    XE_IOCTL_DBG(oa->xe, ext.name >= ARRAY_SIZE(xe_oa_user_extension_funcs)))
1347 		return -EINVAL;
1348 
1349 	idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_oa_user_extension_funcs));
1350 	err = xe_oa_user_extension_funcs[idx](oa, from, extension, param);
1351 	if (XE_IOCTL_DBG(oa->xe, err))
1352 		return err;
1353 
1354 	if (ext.next_extension)
1355 		return xe_oa_user_extensions(oa, from, ext.next_extension, ++ext_number, param);
1356 
1357 	return 0;
1358 }
1359 
xe_oa_parse_syncs(struct xe_oa * oa,struct xe_oa_open_param * param)1360 static int xe_oa_parse_syncs(struct xe_oa *oa, struct xe_oa_open_param *param)
1361 {
1362 	int ret, num_syncs, num_ufence = 0;
1363 
1364 	if (param->num_syncs && !param->syncs_user) {
1365 		drm_dbg(&oa->xe->drm, "num_syncs specified without sync array\n");
1366 		ret = -EINVAL;
1367 		goto exit;
1368 	}
1369 
1370 	if (param->num_syncs) {
1371 		param->syncs = kcalloc(param->num_syncs, sizeof(*param->syncs), GFP_KERNEL);
1372 		if (!param->syncs) {
1373 			ret = -ENOMEM;
1374 			goto exit;
1375 		}
1376 	}
1377 
1378 	for (num_syncs = 0; num_syncs < param->num_syncs; num_syncs++) {
1379 		ret = xe_sync_entry_parse(oa->xe, param->xef, &param->syncs[num_syncs],
1380 					  &param->syncs_user[num_syncs], 0);
1381 		if (ret)
1382 			goto err_syncs;
1383 
1384 		if (xe_sync_is_ufence(&param->syncs[num_syncs]))
1385 			num_ufence++;
1386 	}
1387 
1388 	if (XE_IOCTL_DBG(oa->xe, num_ufence > 1)) {
1389 		ret = -EINVAL;
1390 		goto err_syncs;
1391 	}
1392 
1393 	return 0;
1394 
1395 err_syncs:
1396 	while (num_syncs--)
1397 		xe_sync_entry_cleanup(&param->syncs[num_syncs]);
1398 	kfree(param->syncs);
1399 exit:
1400 	return ret;
1401 }
1402 
xe_oa_stream_enable(struct xe_oa_stream * stream)1403 static void xe_oa_stream_enable(struct xe_oa_stream *stream)
1404 {
1405 	stream->pollin = false;
1406 
1407 	xe_oa_enable(stream);
1408 
1409 	if (stream->sample)
1410 		hrtimer_start(&stream->poll_check_timer,
1411 			      ns_to_ktime(stream->poll_period_ns),
1412 			      HRTIMER_MODE_REL_PINNED);
1413 }
1414 
xe_oa_stream_disable(struct xe_oa_stream * stream)1415 static void xe_oa_stream_disable(struct xe_oa_stream *stream)
1416 {
1417 	xe_oa_disable(stream);
1418 
1419 	if (stream->sample)
1420 		hrtimer_cancel(&stream->poll_check_timer);
1421 }
1422 
xe_oa_enable_preempt_timeslice(struct xe_oa_stream * stream)1423 static int xe_oa_enable_preempt_timeslice(struct xe_oa_stream *stream)
1424 {
1425 	struct xe_exec_queue *q = stream->exec_q;
1426 	int ret1, ret2;
1427 
1428 	/* Best effort recovery: try to revert both to original, irrespective of error */
1429 	ret1 = q->ops->set_timeslice(q, stream->hwe->eclass->sched_props.timeslice_us);
1430 	ret2 = q->ops->set_preempt_timeout(q, stream->hwe->eclass->sched_props.preempt_timeout_us);
1431 	if (ret1 || ret2)
1432 		goto err;
1433 	return 0;
1434 err:
1435 	drm_dbg(&stream->oa->xe->drm, "%s failed ret1 %d ret2 %d\n", __func__, ret1, ret2);
1436 	return ret1 ?: ret2;
1437 }
1438 
xe_oa_disable_preempt_timeslice(struct xe_oa_stream * stream)1439 static int xe_oa_disable_preempt_timeslice(struct xe_oa_stream *stream)
1440 {
1441 	struct xe_exec_queue *q = stream->exec_q;
1442 	int ret;
1443 
1444 	/* Setting values to 0 will disable timeslice and preempt_timeout */
1445 	ret = q->ops->set_timeslice(q, 0);
1446 	if (ret)
1447 		goto err;
1448 
1449 	ret = q->ops->set_preempt_timeout(q, 0);
1450 	if (ret)
1451 		goto err;
1452 
1453 	return 0;
1454 err:
1455 	xe_oa_enable_preempt_timeslice(stream);
1456 	drm_dbg(&stream->oa->xe->drm, "%s failed %d\n", __func__, ret);
1457 	return ret;
1458 }
1459 
xe_oa_enable_locked(struct xe_oa_stream * stream)1460 static int xe_oa_enable_locked(struct xe_oa_stream *stream)
1461 {
1462 	if (stream->enabled)
1463 		return 0;
1464 
1465 	if (stream->no_preempt) {
1466 		int ret = xe_oa_disable_preempt_timeslice(stream);
1467 
1468 		if (ret)
1469 			return ret;
1470 	}
1471 
1472 	xe_oa_stream_enable(stream);
1473 
1474 	stream->enabled = true;
1475 	return 0;
1476 }
1477 
xe_oa_disable_locked(struct xe_oa_stream * stream)1478 static int xe_oa_disable_locked(struct xe_oa_stream *stream)
1479 {
1480 	int ret = 0;
1481 
1482 	if (!stream->enabled)
1483 		return 0;
1484 
1485 	xe_oa_stream_disable(stream);
1486 
1487 	if (stream->no_preempt)
1488 		ret = xe_oa_enable_preempt_timeslice(stream);
1489 
1490 	stream->enabled = false;
1491 	return ret;
1492 }
1493 
xe_oa_config_locked(struct xe_oa_stream * stream,u64 arg)1494 static long xe_oa_config_locked(struct xe_oa_stream *stream, u64 arg)
1495 {
1496 	struct xe_oa_open_param param = {};
1497 	long ret = stream->oa_config->id;
1498 	struct xe_oa_config *config;
1499 	int err;
1500 
1501 	err = xe_oa_user_extensions(stream->oa, XE_OA_USER_EXTN_FROM_CONFIG, arg, 0, &param);
1502 	if (err)
1503 		return err;
1504 
1505 	config = xe_oa_get_oa_config(stream->oa, param.metric_set);
1506 	if (!config)
1507 		return -ENODEV;
1508 
1509 	param.xef = stream->xef;
1510 	err = xe_oa_parse_syncs(stream->oa, &param);
1511 	if (err)
1512 		goto err_config_put;
1513 
1514 	stream->num_syncs = param.num_syncs;
1515 	stream->syncs = param.syncs;
1516 
1517 	err = xe_oa_emit_oa_config(stream, config);
1518 	if (!err) {
1519 		config = xchg(&stream->oa_config, config);
1520 		drm_dbg(&stream->oa->xe->drm, "changed to oa config uuid=%s\n",
1521 			stream->oa_config->uuid);
1522 	}
1523 
1524 err_config_put:
1525 	xe_oa_config_put(config);
1526 
1527 	return err ?: ret;
1528 }
1529 
xe_oa_status_locked(struct xe_oa_stream * stream,unsigned long arg)1530 static long xe_oa_status_locked(struct xe_oa_stream *stream, unsigned long arg)
1531 {
1532 	struct drm_xe_oa_stream_status status = {};
1533 	void __user *uaddr = (void __user *)arg;
1534 
1535 	/* Map from register to uapi bits */
1536 	if (stream->oa_status & OASTATUS_REPORT_LOST)
1537 		status.oa_status |= DRM_XE_OASTATUS_REPORT_LOST;
1538 	if (stream->oa_status & OASTATUS_BUFFER_OVERFLOW)
1539 		status.oa_status |= DRM_XE_OASTATUS_BUFFER_OVERFLOW;
1540 	if (stream->oa_status & OASTATUS_COUNTER_OVERFLOW)
1541 		status.oa_status |= DRM_XE_OASTATUS_COUNTER_OVERFLOW;
1542 	if (stream->oa_status & OASTATUS_MMIO_TRG_Q_FULL)
1543 		status.oa_status |= DRM_XE_OASTATUS_MMIO_TRG_Q_FULL;
1544 
1545 	if (copy_to_user(uaddr, &status, sizeof(status)))
1546 		return -EFAULT;
1547 
1548 	return 0;
1549 }
1550 
xe_oa_info_locked(struct xe_oa_stream * stream,unsigned long arg)1551 static long xe_oa_info_locked(struct xe_oa_stream *stream, unsigned long arg)
1552 {
1553 	struct drm_xe_oa_stream_info info = { .oa_buf_size = stream->oa_buffer.bo->size, };
1554 	void __user *uaddr = (void __user *)arg;
1555 
1556 	if (copy_to_user(uaddr, &info, sizeof(info)))
1557 		return -EFAULT;
1558 
1559 	return 0;
1560 }
1561 
xe_oa_ioctl_locked(struct xe_oa_stream * stream,unsigned int cmd,unsigned long arg)1562 static long xe_oa_ioctl_locked(struct xe_oa_stream *stream,
1563 			       unsigned int cmd,
1564 			       unsigned long arg)
1565 {
1566 	switch (cmd) {
1567 	case DRM_XE_OBSERVATION_IOCTL_ENABLE:
1568 		return xe_oa_enable_locked(stream);
1569 	case DRM_XE_OBSERVATION_IOCTL_DISABLE:
1570 		return xe_oa_disable_locked(stream);
1571 	case DRM_XE_OBSERVATION_IOCTL_CONFIG:
1572 		return xe_oa_config_locked(stream, arg);
1573 	case DRM_XE_OBSERVATION_IOCTL_STATUS:
1574 		return xe_oa_status_locked(stream, arg);
1575 	case DRM_XE_OBSERVATION_IOCTL_INFO:
1576 		return xe_oa_info_locked(stream, arg);
1577 	}
1578 
1579 	return -EINVAL;
1580 }
1581 
xe_oa_ioctl(struct file * file,unsigned int cmd,unsigned long arg)1582 static long xe_oa_ioctl(struct file *file,
1583 			unsigned int cmd,
1584 			unsigned long arg)
1585 {
1586 	struct xe_oa_stream *stream = file->private_data;
1587 	long ret;
1588 
1589 	mutex_lock(&stream->stream_lock);
1590 	ret = xe_oa_ioctl_locked(stream, cmd, arg);
1591 	mutex_unlock(&stream->stream_lock);
1592 
1593 	return ret;
1594 }
1595 
xe_oa_destroy_locked(struct xe_oa_stream * stream)1596 static void xe_oa_destroy_locked(struct xe_oa_stream *stream)
1597 {
1598 	if (stream->enabled)
1599 		xe_oa_disable_locked(stream);
1600 
1601 	xe_oa_stream_destroy(stream);
1602 
1603 	if (stream->exec_q)
1604 		xe_exec_queue_put(stream->exec_q);
1605 
1606 	kfree(stream);
1607 }
1608 
xe_oa_release(struct inode * inode,struct file * file)1609 static int xe_oa_release(struct inode *inode, struct file *file)
1610 {
1611 	struct xe_oa_stream *stream = file->private_data;
1612 	struct xe_gt *gt = stream->gt;
1613 
1614 	xe_pm_runtime_get(gt_to_xe(gt));
1615 	mutex_lock(&gt->oa.gt_lock);
1616 	xe_oa_destroy_locked(stream);
1617 	mutex_unlock(&gt->oa.gt_lock);
1618 	xe_pm_runtime_put(gt_to_xe(gt));
1619 
1620 	/* Release the reference the OA stream kept on the driver */
1621 	drm_dev_put(&gt_to_xe(gt)->drm);
1622 
1623 	return 0;
1624 }
1625 
xe_oa_mmap(struct file * file,struct vm_area_struct * vma)1626 static int xe_oa_mmap(struct file *file, struct vm_area_struct *vma)
1627 {
1628 	struct xe_oa_stream *stream = file->private_data;
1629 	struct xe_bo *bo = stream->oa_buffer.bo;
1630 	unsigned long start = vma->vm_start;
1631 	int i, ret;
1632 
1633 	if (xe_observation_paranoid && !perfmon_capable()) {
1634 		drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n");
1635 		return -EACCES;
1636 	}
1637 
1638 	/* Can mmap the entire OA buffer or nothing (no partial OA buffer mmaps) */
1639 	if (vma->vm_end - vma->vm_start != stream->oa_buffer.bo->size) {
1640 		drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n");
1641 		return -EINVAL;
1642 	}
1643 
1644 	/*
1645 	 * Only support VM_READ, enforce MAP_PRIVATE by checking for
1646 	 * VM_MAYSHARE, don't copy the vma on fork
1647 	 */
1648 	if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_SHARED | VM_MAYSHARE)) {
1649 		drm_dbg(&stream->oa->xe->drm, "mmap must be read only\n");
1650 		return -EINVAL;
1651 	}
1652 	vm_flags_mod(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_DONTCOPY,
1653 		     VM_MAYWRITE | VM_MAYEXEC);
1654 
1655 	xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == vma_pages(vma));
1656 	for (i = 0; i < bo->ttm.ttm->num_pages; i++) {
1657 		ret = remap_pfn_range(vma, start, page_to_pfn(bo->ttm.ttm->pages[i]),
1658 				      PAGE_SIZE, vma->vm_page_prot);
1659 		if (ret)
1660 			break;
1661 
1662 		start += PAGE_SIZE;
1663 	}
1664 
1665 	return ret;
1666 }
1667 
1668 static const struct file_operations xe_oa_fops = {
1669 	.owner		= THIS_MODULE,
1670 	.release	= xe_oa_release,
1671 	.poll		= xe_oa_poll,
1672 	.read		= xe_oa_read,
1673 	.unlocked_ioctl	= xe_oa_ioctl,
1674 	.mmap		= xe_oa_mmap,
1675 };
1676 
xe_oa_stream_init(struct xe_oa_stream * stream,struct xe_oa_open_param * param)1677 static int xe_oa_stream_init(struct xe_oa_stream *stream,
1678 			     struct xe_oa_open_param *param)
1679 {
1680 	struct xe_oa_unit *u = param->hwe->oa_unit;
1681 	struct xe_gt *gt = param->hwe->gt;
1682 	unsigned int fw_ref;
1683 	int ret;
1684 
1685 	stream->exec_q = param->exec_q;
1686 	stream->poll_period_ns = DEFAULT_POLL_PERIOD_NS;
1687 	stream->hwe = param->hwe;
1688 	stream->gt = stream->hwe->gt;
1689 	stream->oa_buffer.format = &stream->oa->oa_formats[param->oa_format];
1690 
1691 	stream->sample = param->sample;
1692 	stream->periodic = param->period_exponent >= 0;
1693 	stream->period_exponent = param->period_exponent;
1694 	stream->no_preempt = param->no_preempt;
1695 	stream->wait_num_reports = param->wait_num_reports;
1696 
1697 	stream->xef = xe_file_get(param->xef);
1698 	stream->num_syncs = param->num_syncs;
1699 	stream->syncs = param->syncs;
1700 
1701 	/*
1702 	 * For Xe2+, when overrun mode is enabled, there are no partial reports at the end
1703 	 * of buffer, making the OA buffer effectively a non-power-of-2 size circular
1704 	 * buffer whose size, circ_size, is a multiple of the report size
1705 	 */
1706 	if (GRAPHICS_VER(stream->oa->xe) >= 20 &&
1707 	    stream->hwe->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG && stream->sample)
1708 		stream->oa_buffer.circ_size =
1709 			param->oa_buffer_size -
1710 			param->oa_buffer_size % stream->oa_buffer.format->size;
1711 	else
1712 		stream->oa_buffer.circ_size = param->oa_buffer_size;
1713 
1714 	stream->oa_config = xe_oa_get_oa_config(stream->oa, param->metric_set);
1715 	if (!stream->oa_config) {
1716 		drm_dbg(&stream->oa->xe->drm, "Invalid OA config id=%i\n", param->metric_set);
1717 		ret = -EINVAL;
1718 		goto exit;
1719 	}
1720 
1721 	/*
1722 	 * Wa_1509372804:pvc
1723 	 *
1724 	 * GuC reset of engines causes OA to lose configuration
1725 	 * state. Prevent this by overriding GUCRC mode.
1726 	 */
1727 	if (stream->oa->xe->info.platform == XE_PVC) {
1728 		ret = xe_guc_pc_override_gucrc_mode(&gt->uc.guc.pc,
1729 						    SLPC_GUCRC_MODE_GUCRC_NO_RC6);
1730 		if (ret)
1731 			goto err_free_configs;
1732 
1733 		stream->override_gucrc = true;
1734 	}
1735 
1736 	/* Take runtime pm ref and forcewake to disable RC6 */
1737 	xe_pm_runtime_get(stream->oa->xe);
1738 	fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL);
1739 	if (!xe_force_wake_ref_has_domain(fw_ref, XE_FORCEWAKE_ALL)) {
1740 		ret = -ETIMEDOUT;
1741 		goto err_fw_put;
1742 	}
1743 
1744 	ret = xe_oa_alloc_oa_buffer(stream, param->oa_buffer_size);
1745 	if (ret)
1746 		goto err_fw_put;
1747 
1748 	stream->k_exec_q = xe_exec_queue_create(stream->oa->xe, NULL,
1749 						BIT(stream->hwe->logical_instance), 1,
1750 						stream->hwe, EXEC_QUEUE_FLAG_KERNEL, 0);
1751 	if (IS_ERR(stream->k_exec_q)) {
1752 		ret = PTR_ERR(stream->k_exec_q);
1753 		drm_err(&stream->oa->xe->drm, "gt%d, hwe %s, xe_exec_queue_create failed=%d",
1754 			stream->gt->info.id, stream->hwe->name, ret);
1755 		goto err_free_oa_buf;
1756 	}
1757 
1758 	ret = xe_oa_enable_metric_set(stream);
1759 	if (ret) {
1760 		drm_dbg(&stream->oa->xe->drm, "Unable to enable metric set\n");
1761 		goto err_put_k_exec_q;
1762 	}
1763 
1764 	drm_dbg(&stream->oa->xe->drm, "opening stream oa config uuid=%s\n",
1765 		stream->oa_config->uuid);
1766 
1767 	WRITE_ONCE(u->exclusive_stream, stream);
1768 
1769 	hrtimer_init(&stream->poll_check_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1770 	stream->poll_check_timer.function = xe_oa_poll_check_timer_cb;
1771 	init_waitqueue_head(&stream->poll_wq);
1772 
1773 	spin_lock_init(&stream->oa_buffer.ptr_lock);
1774 	mutex_init(&stream->stream_lock);
1775 
1776 	return 0;
1777 
1778 err_put_k_exec_q:
1779 	xe_oa_disable_metric_set(stream);
1780 	xe_exec_queue_put(stream->k_exec_q);
1781 err_free_oa_buf:
1782 	xe_oa_free_oa_buffer(stream);
1783 err_fw_put:
1784 	xe_force_wake_put(gt_to_fw(gt), fw_ref);
1785 	xe_pm_runtime_put(stream->oa->xe);
1786 	if (stream->override_gucrc)
1787 		xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(&gt->uc.guc.pc));
1788 err_free_configs:
1789 	xe_oa_free_configs(stream);
1790 exit:
1791 	xe_file_put(stream->xef);
1792 	return ret;
1793 }
1794 
xe_oa_stream_open_ioctl_locked(struct xe_oa * oa,struct xe_oa_open_param * param)1795 static int xe_oa_stream_open_ioctl_locked(struct xe_oa *oa,
1796 					  struct xe_oa_open_param *param)
1797 {
1798 	struct xe_oa_stream *stream;
1799 	int stream_fd;
1800 	int ret;
1801 
1802 	/* We currently only allow exclusive access */
1803 	if (param->hwe->oa_unit->exclusive_stream) {
1804 		drm_dbg(&oa->xe->drm, "OA unit already in use\n");
1805 		ret = -EBUSY;
1806 		goto exit;
1807 	}
1808 
1809 	stream = kzalloc(sizeof(*stream), GFP_KERNEL);
1810 	if (!stream) {
1811 		ret = -ENOMEM;
1812 		goto exit;
1813 	}
1814 
1815 	stream->oa = oa;
1816 	ret = xe_oa_stream_init(stream, param);
1817 	if (ret)
1818 		goto err_free;
1819 
1820 	if (!param->disabled) {
1821 		ret = xe_oa_enable_locked(stream);
1822 		if (ret)
1823 			goto err_destroy;
1824 	}
1825 
1826 	stream_fd = anon_inode_getfd("[xe_oa]", &xe_oa_fops, stream, 0);
1827 	if (stream_fd < 0) {
1828 		ret = stream_fd;
1829 		goto err_disable;
1830 	}
1831 
1832 	/* Hold a reference on the drm device till stream_fd is released */
1833 	drm_dev_get(&stream->oa->xe->drm);
1834 
1835 	return stream_fd;
1836 err_disable:
1837 	if (!param->disabled)
1838 		xe_oa_disable_locked(stream);
1839 err_destroy:
1840 	xe_oa_stream_destroy(stream);
1841 err_free:
1842 	kfree(stream);
1843 exit:
1844 	return ret;
1845 }
1846 
1847 /**
1848  * xe_oa_timestamp_frequency - Return OA timestamp frequency
1849  * @gt: @xe_gt
1850  *
1851  * OA timestamp frequency = CS timestamp frequency in most platforms. On some
1852  * platforms OA unit ignores the CTC_SHIFT and the 2 timestamps differ. In such
1853  * cases, return the adjusted CS timestamp frequency to the user.
1854  */
xe_oa_timestamp_frequency(struct xe_gt * gt)1855 u32 xe_oa_timestamp_frequency(struct xe_gt *gt)
1856 {
1857 	u32 reg, shift;
1858 
1859 	/*
1860 	 * Wa_18013179988:dg2
1861 	 * Wa_14015568240:pvc
1862 	 * Wa_14015846243:mtl
1863 	 */
1864 	switch (gt_to_xe(gt)->info.platform) {
1865 	case XE_DG2:
1866 	case XE_PVC:
1867 	case XE_METEORLAKE:
1868 		xe_pm_runtime_get(gt_to_xe(gt));
1869 		reg = xe_mmio_read32(&gt->mmio, RPM_CONFIG0);
1870 		xe_pm_runtime_put(gt_to_xe(gt));
1871 
1872 		shift = REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, reg);
1873 		return gt->info.reference_clock << (3 - shift);
1874 
1875 	default:
1876 		return gt->info.reference_clock;
1877 	}
1878 }
1879 
oa_exponent_to_ns(struct xe_gt * gt,int exponent)1880 static u64 oa_exponent_to_ns(struct xe_gt *gt, int exponent)
1881 {
1882 	u64 nom = (2ULL << exponent) * NSEC_PER_SEC;
1883 	u32 den = xe_oa_timestamp_frequency(gt);
1884 
1885 	return div_u64(nom + den - 1, den);
1886 }
1887 
engine_supports_oa_format(const struct xe_hw_engine * hwe,int type)1888 static bool engine_supports_oa_format(const struct xe_hw_engine *hwe, int type)
1889 {
1890 	switch (hwe->oa_unit->type) {
1891 	case DRM_XE_OA_UNIT_TYPE_OAG:
1892 		return type == DRM_XE_OA_FMT_TYPE_OAG || type == DRM_XE_OA_FMT_TYPE_OAR ||
1893 			type == DRM_XE_OA_FMT_TYPE_OAC || type == DRM_XE_OA_FMT_TYPE_PEC;
1894 	case DRM_XE_OA_UNIT_TYPE_OAM:
1895 		return type == DRM_XE_OA_FMT_TYPE_OAM || type == DRM_XE_OA_FMT_TYPE_OAM_MPEC;
1896 	default:
1897 		return false;
1898 	}
1899 }
1900 
1901 /**
1902  * xe_oa_unit_id - Return OA unit ID for a hardware engine
1903  * @hwe: @xe_hw_engine
1904  *
1905  * Return OA unit ID for a hardware engine when available
1906  */
xe_oa_unit_id(struct xe_hw_engine * hwe)1907 u16 xe_oa_unit_id(struct xe_hw_engine *hwe)
1908 {
1909 	return hwe->oa_unit && hwe->oa_unit->num_engines ?
1910 		hwe->oa_unit->oa_unit_id : U16_MAX;
1911 }
1912 
xe_oa_assign_hwe(struct xe_oa * oa,struct xe_oa_open_param * param)1913 static int xe_oa_assign_hwe(struct xe_oa *oa, struct xe_oa_open_param *param)
1914 {
1915 	struct xe_gt *gt;
1916 	int i, ret = 0;
1917 
1918 	if (param->exec_q) {
1919 		/* When we have an exec_q, get hwe from the exec_q */
1920 		param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class,
1921 					     param->engine_instance, true);
1922 	} else {
1923 		struct xe_hw_engine *hwe;
1924 		enum xe_hw_engine_id id;
1925 
1926 		/* Else just get the first hwe attached to the oa unit */
1927 		for_each_gt(gt, oa->xe, i) {
1928 			for_each_hw_engine(hwe, gt, id) {
1929 				if (xe_oa_unit_id(hwe) == param->oa_unit_id) {
1930 					param->hwe = hwe;
1931 					goto out;
1932 				}
1933 			}
1934 		}
1935 	}
1936 out:
1937 	if (!param->hwe || xe_oa_unit_id(param->hwe) != param->oa_unit_id) {
1938 		drm_dbg(&oa->xe->drm, "Unable to find hwe (%d, %d) for OA unit ID %d\n",
1939 			param->exec_q ? param->exec_q->class : -1,
1940 			param->engine_instance, param->oa_unit_id);
1941 		ret = -EINVAL;
1942 	}
1943 
1944 	return ret;
1945 }
1946 
1947 /**
1948  * xe_oa_stream_open_ioctl - Opens an OA stream
1949  * @dev: @drm_device
1950  * @data: pointer to struct @drm_xe_oa_config
1951  * @file: @drm_file
1952  *
1953  * The functions opens an OA stream. An OA stream, opened with specified
1954  * properties, enables OA counter samples to be collected, either
1955  * periodically (time based sampling), or on request (using OA queries)
1956  */
xe_oa_stream_open_ioctl(struct drm_device * dev,u64 data,struct drm_file * file)1957 int xe_oa_stream_open_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
1958 {
1959 	struct xe_device *xe = to_xe_device(dev);
1960 	struct xe_oa *oa = &xe->oa;
1961 	struct xe_file *xef = to_xe_file(file);
1962 	struct xe_oa_open_param param = {};
1963 	const struct xe_oa_format *f;
1964 	bool privileged_op = true;
1965 	int ret;
1966 
1967 	if (!oa->xe) {
1968 		drm_dbg(&xe->drm, "xe oa interface not available for this system\n");
1969 		return -ENODEV;
1970 	}
1971 
1972 	param.xef = xef;
1973 	param.period_exponent = -1;
1974 	ret = xe_oa_user_extensions(oa, XE_OA_USER_EXTN_FROM_OPEN, data, 0, &param);
1975 	if (ret)
1976 		return ret;
1977 
1978 	if (param.exec_queue_id > 0) {
1979 		param.exec_q = xe_exec_queue_lookup(xef, param.exec_queue_id);
1980 		if (XE_IOCTL_DBG(oa->xe, !param.exec_q))
1981 			return -ENOENT;
1982 
1983 		if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1))
1984 			return -EOPNOTSUPP;
1985 	}
1986 
1987 	/*
1988 	 * Query based sampling (using MI_REPORT_PERF_COUNT) with OAR/OAC,
1989 	 * without global stream access, can be an unprivileged operation
1990 	 */
1991 	if (param.exec_q && !param.sample)
1992 		privileged_op = false;
1993 
1994 	if (param.no_preempt) {
1995 		if (!param.exec_q) {
1996 			drm_dbg(&oa->xe->drm, "Preemption disable without exec_q!\n");
1997 			ret = -EINVAL;
1998 			goto err_exec_q;
1999 		}
2000 		privileged_op = true;
2001 	}
2002 
2003 	if (privileged_op && xe_observation_paranoid && !perfmon_capable()) {
2004 		drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n");
2005 		ret = -EACCES;
2006 		goto err_exec_q;
2007 	}
2008 
2009 	if (!param.exec_q && !param.sample) {
2010 		drm_dbg(&oa->xe->drm, "Only OA report sampling supported\n");
2011 		ret = -EINVAL;
2012 		goto err_exec_q;
2013 	}
2014 
2015 	ret = xe_oa_assign_hwe(oa, &param);
2016 	if (ret)
2017 		goto err_exec_q;
2018 
2019 	f = &oa->oa_formats[param.oa_format];
2020 	if (!param.oa_format || !f->size ||
2021 	    !engine_supports_oa_format(param.hwe, f->type)) {
2022 		drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n",
2023 			param.oa_format, f->type, f->size, param.hwe->class);
2024 		ret = -EINVAL;
2025 		goto err_exec_q;
2026 	}
2027 
2028 	if (param.period_exponent >= 0) {
2029 		u64 oa_period, oa_freq_hz;
2030 
2031 		/* Requesting samples from OAG buffer is a privileged operation */
2032 		if (!param.sample) {
2033 			drm_dbg(&oa->xe->drm, "OA_EXPONENT specified without SAMPLE_OA\n");
2034 			ret = -EINVAL;
2035 			goto err_exec_q;
2036 		}
2037 		oa_period = oa_exponent_to_ns(param.hwe->gt, param.period_exponent);
2038 		oa_freq_hz = div64_u64(NSEC_PER_SEC, oa_period);
2039 		drm_dbg(&oa->xe->drm, "Using periodic sampling freq %lld Hz\n", oa_freq_hz);
2040 	}
2041 
2042 	if (!param.oa_buffer_size)
2043 		param.oa_buffer_size = DEFAULT_XE_OA_BUFFER_SIZE;
2044 
2045 	if (!param.wait_num_reports)
2046 		param.wait_num_reports = 1;
2047 	if (param.wait_num_reports > param.oa_buffer_size / f->size) {
2048 		drm_dbg(&oa->xe->drm, "wait_num_reports %d\n", param.wait_num_reports);
2049 		ret = -EINVAL;
2050 		goto err_exec_q;
2051 	}
2052 
2053 	ret = xe_oa_parse_syncs(oa, &param);
2054 	if (ret)
2055 		goto err_exec_q;
2056 
2057 	mutex_lock(&param.hwe->gt->oa.gt_lock);
2058 	ret = xe_oa_stream_open_ioctl_locked(oa, &param);
2059 	mutex_unlock(&param.hwe->gt->oa.gt_lock);
2060 	if (ret < 0)
2061 		goto err_sync_cleanup;
2062 
2063 	return ret;
2064 
2065 err_sync_cleanup:
2066 	while (param.num_syncs--)
2067 		xe_sync_entry_cleanup(&param.syncs[param.num_syncs]);
2068 	kfree(param.syncs);
2069 err_exec_q:
2070 	if (param.exec_q)
2071 		xe_exec_queue_put(param.exec_q);
2072 	return ret;
2073 }
2074 
xe_oa_is_valid_flex_addr(struct xe_oa * oa,u32 addr)2075 static bool xe_oa_is_valid_flex_addr(struct xe_oa *oa, u32 addr)
2076 {
2077 	static const struct xe_reg flex_eu_regs[] = {
2078 		EU_PERF_CNTL0,
2079 		EU_PERF_CNTL1,
2080 		EU_PERF_CNTL2,
2081 		EU_PERF_CNTL3,
2082 		EU_PERF_CNTL4,
2083 		EU_PERF_CNTL5,
2084 		EU_PERF_CNTL6,
2085 	};
2086 	int i;
2087 
2088 	for (i = 0; i < ARRAY_SIZE(flex_eu_regs); i++) {
2089 		if (flex_eu_regs[i].addr == addr)
2090 			return true;
2091 	}
2092 	return false;
2093 }
2094 
xe_oa_reg_in_range_table(u32 addr,const struct xe_mmio_range * table)2095 static bool xe_oa_reg_in_range_table(u32 addr, const struct xe_mmio_range *table)
2096 {
2097 	while (table->start && table->end) {
2098 		if (addr >= table->start && addr <= table->end)
2099 			return true;
2100 
2101 		table++;
2102 	}
2103 
2104 	return false;
2105 }
2106 
2107 static const struct xe_mmio_range xehp_oa_b_counters[] = {
2108 	{ .start = 0xdc48, .end = 0xdc48 },	/* OAA_ENABLE_REG */
2109 	{ .start = 0xdd00, .end = 0xdd48 },	/* OAG_LCE0_0 - OAA_LENABLE_REG */
2110 	{}
2111 };
2112 
2113 static const struct xe_mmio_range gen12_oa_b_counters[] = {
2114 	{ .start = 0x2b2c, .end = 0x2b2c },	/* OAG_OA_PESS */
2115 	{ .start = 0xd900, .end = 0xd91c },	/* OAG_OASTARTTRIG[1-8] */
2116 	{ .start = 0xd920, .end = 0xd93c },	/* OAG_OAREPORTTRIG1[1-8] */
2117 	{ .start = 0xd940, .end = 0xd97c },	/* OAG_CEC[0-7][0-1] */
2118 	{ .start = 0xdc00, .end = 0xdc3c },	/* OAG_SCEC[0-7][0-1] */
2119 	{ .start = 0xdc40, .end = 0xdc40 },	/* OAG_SPCTR_CNF */
2120 	{ .start = 0xdc44, .end = 0xdc44 },	/* OAA_DBG_REG */
2121 	{}
2122 };
2123 
2124 static const struct xe_mmio_range mtl_oam_b_counters[] = {
2125 	{ .start = 0x393000, .end = 0x39301c },	/* OAM_STARTTRIG1[1-8] */
2126 	{ .start = 0x393020, .end = 0x39303c },	/* OAM_REPORTTRIG1[1-8] */
2127 	{ .start = 0x393040, .end = 0x39307c },	/* OAM_CEC[0-7][0-1] */
2128 	{ .start = 0x393200, .end = 0x39323C },	/* MPES[0-7] */
2129 	{}
2130 };
2131 
2132 static const struct xe_mmio_range xe2_oa_b_counters[] = {
2133 	{ .start = 0x393200, .end = 0x39323C },	/* MPES_0_MPES_SAG - MPES_7_UPPER_MPES_SAG */
2134 	{ .start = 0x394200, .end = 0x39423C },	/* MPES_0_MPES_SCMI0 - MPES_7_UPPER_MPES_SCMI0 */
2135 	{ .start = 0x394A00, .end = 0x394A3C },	/* MPES_0_MPES_SCMI1 - MPES_7_UPPER_MPES_SCMI1 */
2136 	{},
2137 };
2138 
xe_oa_is_valid_b_counter_addr(struct xe_oa * oa,u32 addr)2139 static bool xe_oa_is_valid_b_counter_addr(struct xe_oa *oa, u32 addr)
2140 {
2141 	return xe_oa_reg_in_range_table(addr, xehp_oa_b_counters) ||
2142 		xe_oa_reg_in_range_table(addr, gen12_oa_b_counters) ||
2143 		xe_oa_reg_in_range_table(addr, mtl_oam_b_counters) ||
2144 		(GRAPHICS_VER(oa->xe) >= 20 &&
2145 		 xe_oa_reg_in_range_table(addr, xe2_oa_b_counters));
2146 }
2147 
2148 static const struct xe_mmio_range mtl_oa_mux_regs[] = {
2149 	{ .start = 0x0d00, .end = 0x0d04 },	/* RPM_CONFIG[0-1] */
2150 	{ .start = 0x0d0c, .end = 0x0d2c },	/* NOA_CONFIG[0-8] */
2151 	{ .start = 0x9840, .end = 0x9840 },	/* GDT_CHICKEN_BITS */
2152 	{ .start = 0x9884, .end = 0x9888 },	/* NOA_WRITE */
2153 	{ .start = 0x38d100, .end = 0x38d114},	/* VISACTL */
2154 	{}
2155 };
2156 
2157 static const struct xe_mmio_range gen12_oa_mux_regs[] = {
2158 	{ .start = 0x0d00, .end = 0x0d04 },     /* RPM_CONFIG[0-1] */
2159 	{ .start = 0x0d0c, .end = 0x0d2c },     /* NOA_CONFIG[0-8] */
2160 	{ .start = 0x9840, .end = 0x9840 },	/* GDT_CHICKEN_BITS */
2161 	{ .start = 0x9884, .end = 0x9888 },	/* NOA_WRITE */
2162 	{ .start = 0x20cc, .end = 0x20cc },	/* WAIT_FOR_RC6_EXIT */
2163 	{}
2164 };
2165 
2166 static const struct xe_mmio_range xe2_oa_mux_regs[] = {
2167 	{ .start = 0x5194, .end = 0x5194 },	/* SYS_MEM_LAT_MEASURE_MERTF_GRP_3D */
2168 	{ .start = 0x8704, .end = 0x8704 },	/* LMEM_LAT_MEASURE_MCFG_GRP */
2169 	{ .start = 0xB1BC, .end = 0xB1BC },	/* L3_BANK_LAT_MEASURE_LBCF_GFX */
2170 	{ .start = 0xD0E0, .end = 0xD0F4 },	/* VISACTL */
2171 	{ .start = 0xE18C, .end = 0xE18C },	/* SAMPLER_MODE */
2172 	{ .start = 0xE590, .end = 0xE590 },	/* TDL_LSC_LAT_MEASURE_TDL_GFX */
2173 	{ .start = 0x13000, .end = 0x137FC },	/* PES_0_PESL0 - PES_63_UPPER_PESL3 */
2174 	{},
2175 };
2176 
xe_oa_is_valid_mux_addr(struct xe_oa * oa,u32 addr)2177 static bool xe_oa_is_valid_mux_addr(struct xe_oa *oa, u32 addr)
2178 {
2179 	if (GRAPHICS_VER(oa->xe) >= 20)
2180 		return xe_oa_reg_in_range_table(addr, xe2_oa_mux_regs);
2181 	else if (GRAPHICS_VERx100(oa->xe) >= 1270)
2182 		return xe_oa_reg_in_range_table(addr, mtl_oa_mux_regs);
2183 	else
2184 		return xe_oa_reg_in_range_table(addr, gen12_oa_mux_regs);
2185 }
2186 
xe_oa_is_valid_config_reg_addr(struct xe_oa * oa,u32 addr)2187 static bool xe_oa_is_valid_config_reg_addr(struct xe_oa *oa, u32 addr)
2188 {
2189 	return xe_oa_is_valid_flex_addr(oa, addr) ||
2190 		xe_oa_is_valid_b_counter_addr(oa, addr) ||
2191 		xe_oa_is_valid_mux_addr(oa, addr);
2192 }
2193 
2194 static struct xe_oa_reg *
xe_oa_alloc_regs(struct xe_oa * oa,bool (* is_valid)(struct xe_oa * oa,u32 addr),u32 __user * regs,u32 n_regs)2195 xe_oa_alloc_regs(struct xe_oa *oa, bool (*is_valid)(struct xe_oa *oa, u32 addr),
2196 		 u32 __user *regs, u32 n_regs)
2197 {
2198 	struct xe_oa_reg *oa_regs;
2199 	int err;
2200 	u32 i;
2201 
2202 	oa_regs = kmalloc_array(n_regs, sizeof(*oa_regs), GFP_KERNEL);
2203 	if (!oa_regs)
2204 		return ERR_PTR(-ENOMEM);
2205 
2206 	for (i = 0; i < n_regs; i++) {
2207 		u32 addr, value;
2208 
2209 		err = get_user(addr, regs);
2210 		if (err)
2211 			goto addr_err;
2212 
2213 		if (!is_valid(oa, addr)) {
2214 			drm_dbg(&oa->xe->drm, "Invalid oa_reg address: %X\n", addr);
2215 			err = -EINVAL;
2216 			goto addr_err;
2217 		}
2218 
2219 		err = get_user(value, regs + 1);
2220 		if (err)
2221 			goto addr_err;
2222 
2223 		oa_regs[i].addr = XE_REG(addr);
2224 		oa_regs[i].value = value;
2225 
2226 		regs += 2;
2227 	}
2228 
2229 	return oa_regs;
2230 
2231 addr_err:
2232 	kfree(oa_regs);
2233 	return ERR_PTR(err);
2234 }
2235 
show_dynamic_id(struct kobject * kobj,struct kobj_attribute * attr,char * buf)2236 static ssize_t show_dynamic_id(struct kobject *kobj,
2237 			       struct kobj_attribute *attr,
2238 			       char *buf)
2239 {
2240 	struct xe_oa_config *oa_config =
2241 		container_of(attr, typeof(*oa_config), sysfs_metric_id);
2242 
2243 	return sysfs_emit(buf, "%d\n", oa_config->id);
2244 }
2245 
create_dynamic_oa_sysfs_entry(struct xe_oa * oa,struct xe_oa_config * oa_config)2246 static int create_dynamic_oa_sysfs_entry(struct xe_oa *oa,
2247 					 struct xe_oa_config *oa_config)
2248 {
2249 	sysfs_attr_init(&oa_config->sysfs_metric_id.attr);
2250 	oa_config->sysfs_metric_id.attr.name = "id";
2251 	oa_config->sysfs_metric_id.attr.mode = 0444;
2252 	oa_config->sysfs_metric_id.show = show_dynamic_id;
2253 	oa_config->sysfs_metric_id.store = NULL;
2254 
2255 	oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr;
2256 	oa_config->attrs[1] = NULL;
2257 
2258 	oa_config->sysfs_metric.name = oa_config->uuid;
2259 	oa_config->sysfs_metric.attrs = oa_config->attrs;
2260 
2261 	return sysfs_create_group(oa->metrics_kobj, &oa_config->sysfs_metric);
2262 }
2263 
2264 /**
2265  * xe_oa_add_config_ioctl - Adds one OA config
2266  * @dev: @drm_device
2267  * @data: pointer to struct @drm_xe_oa_config
2268  * @file: @drm_file
2269  *
2270  * The functions adds an OA config to the set of OA configs maintained in
2271  * the kernel. The config determines which OA metrics are collected for an
2272  * OA stream.
2273  */
xe_oa_add_config_ioctl(struct drm_device * dev,u64 data,struct drm_file * file)2274 int xe_oa_add_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
2275 {
2276 	struct xe_device *xe = to_xe_device(dev);
2277 	struct xe_oa *oa = &xe->oa;
2278 	struct drm_xe_oa_config param;
2279 	struct drm_xe_oa_config *arg = &param;
2280 	struct xe_oa_config *oa_config, *tmp;
2281 	struct xe_oa_reg *regs;
2282 	int err, id;
2283 
2284 	if (!oa->xe) {
2285 		drm_dbg(&xe->drm, "xe oa interface not available for this system\n");
2286 		return -ENODEV;
2287 	}
2288 
2289 	if (xe_observation_paranoid && !perfmon_capable()) {
2290 		drm_dbg(&oa->xe->drm, "Insufficient privileges to add xe OA config\n");
2291 		return -EACCES;
2292 	}
2293 
2294 	err = __copy_from_user(&param, u64_to_user_ptr(data), sizeof(param));
2295 	if (XE_IOCTL_DBG(oa->xe, err))
2296 		return -EFAULT;
2297 
2298 	if (XE_IOCTL_DBG(oa->xe, arg->extensions) ||
2299 	    XE_IOCTL_DBG(oa->xe, !arg->regs_ptr) ||
2300 	    XE_IOCTL_DBG(oa->xe, !arg->n_regs))
2301 		return -EINVAL;
2302 
2303 	oa_config = kzalloc(sizeof(*oa_config), GFP_KERNEL);
2304 	if (!oa_config)
2305 		return -ENOMEM;
2306 
2307 	oa_config->oa = oa;
2308 	kref_init(&oa_config->ref);
2309 
2310 	if (!uuid_is_valid(arg->uuid)) {
2311 		drm_dbg(&oa->xe->drm, "Invalid uuid format for OA config\n");
2312 		err = -EINVAL;
2313 		goto reg_err;
2314 	}
2315 
2316 	/* Last character in oa_config->uuid will be 0 because oa_config is kzalloc */
2317 	memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid));
2318 
2319 	oa_config->regs_len = arg->n_regs;
2320 	regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg_addr,
2321 				u64_to_user_ptr(arg->regs_ptr),
2322 				arg->n_regs);
2323 	if (IS_ERR(regs)) {
2324 		drm_dbg(&oa->xe->drm, "Failed to create OA config for mux_regs\n");
2325 		err = PTR_ERR(regs);
2326 		goto reg_err;
2327 	}
2328 	oa_config->regs = regs;
2329 
2330 	err = mutex_lock_interruptible(&oa->metrics_lock);
2331 	if (err)
2332 		goto reg_err;
2333 
2334 	/* We shouldn't have too many configs, so this iteration shouldn't be too costly */
2335 	idr_for_each_entry(&oa->metrics_idr, tmp, id) {
2336 		if (!strcmp(tmp->uuid, oa_config->uuid)) {
2337 			drm_dbg(&oa->xe->drm, "OA config already exists with this uuid\n");
2338 			err = -EADDRINUSE;
2339 			goto sysfs_err;
2340 		}
2341 	}
2342 
2343 	err = create_dynamic_oa_sysfs_entry(oa, oa_config);
2344 	if (err) {
2345 		drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n");
2346 		goto sysfs_err;
2347 	}
2348 
2349 	oa_config->id = idr_alloc(&oa->metrics_idr, oa_config, 1, 0, GFP_KERNEL);
2350 	if (oa_config->id < 0) {
2351 		drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n");
2352 		err = oa_config->id;
2353 		goto sysfs_err;
2354 	}
2355 
2356 	mutex_unlock(&oa->metrics_lock);
2357 
2358 	drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, oa_config->id);
2359 
2360 	return oa_config->id;
2361 
2362 sysfs_err:
2363 	mutex_unlock(&oa->metrics_lock);
2364 reg_err:
2365 	xe_oa_config_put(oa_config);
2366 	drm_dbg(&oa->xe->drm, "Failed to add new OA config\n");
2367 	return err;
2368 }
2369 
2370 /**
2371  * xe_oa_remove_config_ioctl - Removes one OA config
2372  * @dev: @drm_device
2373  * @data: pointer to struct @drm_xe_observation_param
2374  * @file: @drm_file
2375  */
xe_oa_remove_config_ioctl(struct drm_device * dev,u64 data,struct drm_file * file)2376 int xe_oa_remove_config_ioctl(struct drm_device *dev, u64 data, struct drm_file *file)
2377 {
2378 	struct xe_device *xe = to_xe_device(dev);
2379 	struct xe_oa *oa = &xe->oa;
2380 	struct xe_oa_config *oa_config;
2381 	u64 arg, *ptr = u64_to_user_ptr(data);
2382 	int ret;
2383 
2384 	if (!oa->xe) {
2385 		drm_dbg(&xe->drm, "xe oa interface not available for this system\n");
2386 		return -ENODEV;
2387 	}
2388 
2389 	if (xe_observation_paranoid && !perfmon_capable()) {
2390 		drm_dbg(&oa->xe->drm, "Insufficient privileges to remove xe OA config\n");
2391 		return -EACCES;
2392 	}
2393 
2394 	ret = get_user(arg, ptr);
2395 	if (XE_IOCTL_DBG(oa->xe, ret))
2396 		return ret;
2397 
2398 	ret = mutex_lock_interruptible(&oa->metrics_lock);
2399 	if (ret)
2400 		return ret;
2401 
2402 	oa_config = idr_find(&oa->metrics_idr, arg);
2403 	if (!oa_config) {
2404 		drm_dbg(&oa->xe->drm, "Failed to remove unknown OA config\n");
2405 		ret = -ENOENT;
2406 		goto err_unlock;
2407 	}
2408 
2409 	WARN_ON(arg != oa_config->id);
2410 
2411 	sysfs_remove_group(oa->metrics_kobj, &oa_config->sysfs_metric);
2412 	idr_remove(&oa->metrics_idr, arg);
2413 
2414 	mutex_unlock(&oa->metrics_lock);
2415 
2416 	drm_dbg(&oa->xe->drm, "Removed config %s id=%i\n", oa_config->uuid, oa_config->id);
2417 
2418 	xe_oa_config_put(oa_config);
2419 
2420 	return 0;
2421 
2422 err_unlock:
2423 	mutex_unlock(&oa->metrics_lock);
2424 	return ret;
2425 }
2426 
2427 /**
2428  * xe_oa_register - Xe OA registration
2429  * @xe: @xe_device
2430  *
2431  * Exposes the metrics sysfs directory upon completion of module initialization
2432  */
xe_oa_register(struct xe_device * xe)2433 void xe_oa_register(struct xe_device *xe)
2434 {
2435 	struct xe_oa *oa = &xe->oa;
2436 
2437 	if (!oa->xe)
2438 		return;
2439 
2440 	oa->metrics_kobj = kobject_create_and_add("metrics",
2441 						  &xe->drm.primary->kdev->kobj);
2442 }
2443 
2444 /**
2445  * xe_oa_unregister - Xe OA de-registration
2446  * @xe: @xe_device
2447  */
xe_oa_unregister(struct xe_device * xe)2448 void xe_oa_unregister(struct xe_device *xe)
2449 {
2450 	struct xe_oa *oa = &xe->oa;
2451 
2452 	if (!oa->metrics_kobj)
2453 		return;
2454 
2455 	kobject_put(oa->metrics_kobj);
2456 	oa->metrics_kobj = NULL;
2457 }
2458 
num_oa_units_per_gt(struct xe_gt * gt)2459 static u32 num_oa_units_per_gt(struct xe_gt *gt)
2460 {
2461 	return 1;
2462 }
2463 
__hwe_oam_unit(struct xe_hw_engine * hwe)2464 static u32 __hwe_oam_unit(struct xe_hw_engine *hwe)
2465 {
2466 	if (GRAPHICS_VERx100(gt_to_xe(hwe->gt)) >= 1270) {
2467 		/*
2468 		 * There's 1 SAMEDIA gt and 1 OAM per SAMEDIA gt. All media slices
2469 		 * within the gt use the same OAM. All MTL/LNL SKUs list 1 SA MEDIA
2470 		 */
2471 		xe_gt_WARN_ON(hwe->gt, hwe->gt->info.type != XE_GT_TYPE_MEDIA);
2472 
2473 		return 0;
2474 	}
2475 
2476 	return XE_OA_UNIT_INVALID;
2477 }
2478 
__hwe_oa_unit(struct xe_hw_engine * hwe)2479 static u32 __hwe_oa_unit(struct xe_hw_engine *hwe)
2480 {
2481 	switch (hwe->class) {
2482 	case XE_ENGINE_CLASS_RENDER:
2483 	case XE_ENGINE_CLASS_COMPUTE:
2484 		return 0;
2485 
2486 	case XE_ENGINE_CLASS_VIDEO_DECODE:
2487 	case XE_ENGINE_CLASS_VIDEO_ENHANCE:
2488 		return __hwe_oam_unit(hwe);
2489 
2490 	default:
2491 		return XE_OA_UNIT_INVALID;
2492 	}
2493 }
2494 
__oam_regs(u32 base)2495 static struct xe_oa_regs __oam_regs(u32 base)
2496 {
2497 	return (struct xe_oa_regs) {
2498 		base,
2499 		OAM_HEAD_POINTER(base),
2500 		OAM_TAIL_POINTER(base),
2501 		OAM_BUFFER(base),
2502 		OAM_CONTEXT_CONTROL(base),
2503 		OAM_CONTROL(base),
2504 		OAM_DEBUG(base),
2505 		OAM_STATUS(base),
2506 		OAM_CONTROL_COUNTER_SEL_MASK,
2507 	};
2508 }
2509 
__oag_regs(void)2510 static struct xe_oa_regs __oag_regs(void)
2511 {
2512 	return (struct xe_oa_regs) {
2513 		0,
2514 		OAG_OAHEADPTR,
2515 		OAG_OATAILPTR,
2516 		OAG_OABUFFER,
2517 		OAG_OAGLBCTXCTRL,
2518 		OAG_OACONTROL,
2519 		OAG_OA_DEBUG,
2520 		OAG_OASTATUS,
2521 		OAG_OACONTROL_OA_COUNTER_SEL_MASK,
2522 	};
2523 }
2524 
__xe_oa_init_oa_units(struct xe_gt * gt)2525 static void __xe_oa_init_oa_units(struct xe_gt *gt)
2526 {
2527 	const u32 mtl_oa_base[] = { 0x13000 };
2528 	int i, num_units = gt->oa.num_oa_units;
2529 
2530 	for (i = 0; i < num_units; i++) {
2531 		struct xe_oa_unit *u = &gt->oa.oa_unit[i];
2532 
2533 		if (gt->info.type != XE_GT_TYPE_MEDIA) {
2534 			u->regs = __oag_regs();
2535 			u->type = DRM_XE_OA_UNIT_TYPE_OAG;
2536 		} else if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) {
2537 			u->regs = __oam_regs(mtl_oa_base[i]);
2538 			u->type = DRM_XE_OA_UNIT_TYPE_OAM;
2539 		}
2540 
2541 		xe_mmio_write32(&gt->mmio, u->regs.oa_ctrl, 0);
2542 
2543 		/* Ensure MMIO trigger remains disabled till there is a stream */
2544 		xe_mmio_write32(&gt->mmio, u->regs.oa_debug,
2545 				oag_configure_mmio_trigger(NULL, false));
2546 
2547 		/* Set oa_unit_ids now to ensure ids remain contiguous */
2548 		u->oa_unit_id = gt_to_xe(gt)->oa.oa_unit_ids++;
2549 	}
2550 }
2551 
xe_oa_init_gt(struct xe_gt * gt)2552 static int xe_oa_init_gt(struct xe_gt *gt)
2553 {
2554 	u32 num_oa_units = num_oa_units_per_gt(gt);
2555 	struct xe_hw_engine *hwe;
2556 	enum xe_hw_engine_id id;
2557 	struct xe_oa_unit *u;
2558 
2559 	u = drmm_kcalloc(&gt_to_xe(gt)->drm, num_oa_units, sizeof(*u), GFP_KERNEL);
2560 	if (!u)
2561 		return -ENOMEM;
2562 
2563 	for_each_hw_engine(hwe, gt, id) {
2564 		u32 index = __hwe_oa_unit(hwe);
2565 
2566 		hwe->oa_unit = NULL;
2567 		if (index < num_oa_units) {
2568 			u[index].num_engines++;
2569 			hwe->oa_unit = &u[index];
2570 		}
2571 	}
2572 
2573 	/*
2574 	 * Fused off engines can result in oa_unit's with num_engines == 0. These units
2575 	 * will appear in OA unit query, but no OA streams can be opened on them.
2576 	 */
2577 	gt->oa.num_oa_units = num_oa_units;
2578 	gt->oa.oa_unit = u;
2579 
2580 	__xe_oa_init_oa_units(gt);
2581 
2582 	drmm_mutex_init(&gt_to_xe(gt)->drm, &gt->oa.gt_lock);
2583 
2584 	return 0;
2585 }
2586 
xe_oa_init_oa_units(struct xe_oa * oa)2587 static int xe_oa_init_oa_units(struct xe_oa *oa)
2588 {
2589 	struct xe_gt *gt;
2590 	int i, ret;
2591 
2592 	for_each_gt(gt, oa->xe, i) {
2593 		ret = xe_oa_init_gt(gt);
2594 		if (ret)
2595 			return ret;
2596 	}
2597 
2598 	return 0;
2599 }
2600 
oa_format_add(struct xe_oa * oa,enum xe_oa_format_name format)2601 static void oa_format_add(struct xe_oa *oa, enum xe_oa_format_name format)
2602 {
2603 	__set_bit(format, oa->format_mask);
2604 }
2605 
xe_oa_init_supported_formats(struct xe_oa * oa)2606 static void xe_oa_init_supported_formats(struct xe_oa *oa)
2607 {
2608 	if (GRAPHICS_VER(oa->xe) >= 20) {
2609 		/* Xe2+ */
2610 		oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8);
2611 		oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8);
2612 		oa_format_add(oa, XE_OA_FORMAT_PEC64u64);
2613 		oa_format_add(oa, XE_OA_FORMAT_PEC64u64_B8_C8);
2614 		oa_format_add(oa, XE_OA_FORMAT_PEC64u32);
2615 		oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G1);
2616 		oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G1);
2617 		oa_format_add(oa, XE_OA_FORMAT_PEC32u64_G2);
2618 		oa_format_add(oa, XE_OA_FORMAT_PEC32u32_G2);
2619 		oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_32_G2_4);
2620 		oa_format_add(oa, XE_OA_FORMAT_PEC36u64_G1_4_G2_32);
2621 	} else if (GRAPHICS_VERx100(oa->xe) >= 1270) {
2622 		/* XE_METEORLAKE */
2623 		oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8);
2624 		oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8);
2625 		oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8);
2626 		oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8);
2627 		oa_format_add(oa, XE_OAM_FORMAT_MPEC8u64_B8_C8);
2628 		oa_format_add(oa, XE_OAM_FORMAT_MPEC8u32_B8_C8);
2629 	} else if (GRAPHICS_VERx100(oa->xe) >= 1255) {
2630 		/* XE_DG2, XE_PVC */
2631 		oa_format_add(oa, XE_OAR_FORMAT_A32u40_A4u32_B8_C8);
2632 		oa_format_add(oa, XE_OA_FORMAT_A24u40_A14u32_B8_C8);
2633 		oa_format_add(oa, XE_OAC_FORMAT_A24u64_B8_C8);
2634 		oa_format_add(oa, XE_OAC_FORMAT_A22u32_R2u32_B8_C8);
2635 	} else {
2636 		/* Gen12+ */
2637 		xe_assert(oa->xe, GRAPHICS_VER(oa->xe) >= 12);
2638 		oa_format_add(oa, XE_OA_FORMAT_A12);
2639 		oa_format_add(oa, XE_OA_FORMAT_A12_B8_C8);
2640 		oa_format_add(oa, XE_OA_FORMAT_A32u40_A4u32_B8_C8);
2641 		oa_format_add(oa, XE_OA_FORMAT_C4_B8);
2642 	}
2643 }
2644 
2645 /**
2646  * xe_oa_init - OA initialization during device probe
2647  * @xe: @xe_device
2648  *
2649  * Return: 0 on success or a negative error code on failure
2650  */
xe_oa_init(struct xe_device * xe)2651 int xe_oa_init(struct xe_device *xe)
2652 {
2653 	struct xe_oa *oa = &xe->oa;
2654 	int ret;
2655 
2656 	/* Support OA only with GuC submission and Gen12+ */
2657 	if (!xe_device_uc_enabled(xe) || GRAPHICS_VER(xe) < 12)
2658 		return 0;
2659 
2660 	if (IS_SRIOV_VF(xe))
2661 		return 0;
2662 
2663 	oa->xe = xe;
2664 	oa->oa_formats = oa_formats;
2665 
2666 	drmm_mutex_init(&oa->xe->drm, &oa->metrics_lock);
2667 	idr_init_base(&oa->metrics_idr, 1);
2668 
2669 	ret = xe_oa_init_oa_units(oa);
2670 	if (ret) {
2671 		drm_err(&xe->drm, "OA initialization failed (%pe)\n", ERR_PTR(ret));
2672 		goto exit;
2673 	}
2674 
2675 	xe_oa_init_supported_formats(oa);
2676 	return 0;
2677 exit:
2678 	oa->xe = NULL;
2679 	return ret;
2680 }
2681 
destroy_config(int id,void * p,void * data)2682 static int destroy_config(int id, void *p, void *data)
2683 {
2684 	xe_oa_config_put(p);
2685 	return 0;
2686 }
2687 
2688 /**
2689  * xe_oa_fini - OA de-initialization during device remove
2690  * @xe: @xe_device
2691  */
xe_oa_fini(struct xe_device * xe)2692 void xe_oa_fini(struct xe_device *xe)
2693 {
2694 	struct xe_oa *oa = &xe->oa;
2695 
2696 	if (!oa->xe)
2697 		return;
2698 
2699 	idr_for_each(&oa->metrics_idr, destroy_config, oa);
2700 	idr_destroy(&oa->metrics_idr);
2701 
2702 	oa->xe = NULL;
2703 }
2704