Lines Matching +full:ext +full:- +full:regs
1 // SPDX-License-Identifier: MIT
3 * Copyright © 2023-2024 Intel Corporation
19 #include "regs/xe_engine_regs.h"
20 #include "regs/xe_gt_regs.h"
21 #include "regs/xe_oa_regs.h"
73 const struct xe_oa_reg *regs; member
148 return tail >= head ? tail - head : in xe_oa_circ_diff()
149 tail + stream->oa_buffer.circ_size - head; in xe_oa_circ_diff()
154 return ptr + n >= stream->oa_buffer.circ_size ? in xe_oa_circ_incr()
155 ptr + n - stream->oa_buffer.circ_size : ptr + n; in xe_oa_circ_incr()
163 kfree(oa_config->regs); in xe_oa_config_release()
173 kref_put(&oa_config->ref, xe_oa_config_release); in xe_oa_config_put()
178 return kref_get_unless_zero(&oa_config->ref) ? oa_config : NULL; in xe_oa_config_get()
186 oa_config = idr_find(&oa->metrics_idr, metrics_set); in xe_oa_get_oa_config()
196 xe_oa_config_put(oa_bo->oa_config); in free_oa_config_bo()
197 xe_bb_free(oa_bo->bb, last_fence); in free_oa_config_bo()
203 return &stream->oa_unit->regs; in __oa_regs()
208 return xe_mmio_read32(&stream->gt->mmio, __oa_regs(stream)->oa_tail_ptr) & in xe_oa_hw_tail_read()
213 ((__s)->oa_buffer.format->header == HDR_64_BIT)
245 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); in xe_oa_buffer_check_unlocked()
247 int report_size = stream->oa_buffer.format->size; in xe_oa_buffer_check_unlocked()
250 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_buffer_check_unlocked()
253 hw_tail -= gtt_offset; in xe_oa_buffer_check_unlocked()
260 partial_report_size = xe_oa_circ_diff(stream, hw_tail, stream->oa_buffer.tail); in xe_oa_buffer_check_unlocked()
276 while (xe_oa_circ_diff(stream, tail, stream->oa_buffer.tail) >= report_size) { in xe_oa_buffer_check_unlocked()
277 void *report = stream->oa_buffer.vaddr + tail; in xe_oa_buffer_check_unlocked()
286 drm_dbg(&stream->oa->xe->drm, in xe_oa_buffer_check_unlocked()
288 stream->oa_buffer.head, tail, hw_tail); in xe_oa_buffer_check_unlocked()
290 stream->oa_buffer.tail = tail; in xe_oa_buffer_check_unlocked()
292 available = xe_oa_circ_diff(stream, stream->oa_buffer.tail, stream->oa_buffer.head); in xe_oa_buffer_check_unlocked()
293 stream->pollin = available >= stream->wait_num_reports * report_size; in xe_oa_buffer_check_unlocked()
295 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_buffer_check_unlocked()
297 return stream->pollin; in xe_oa_buffer_check_unlocked()
306 wake_up(&stream->poll_wq); in xe_oa_poll_check_timer_cb()
308 hrtimer_forward_now(hrtimer, ns_to_ktime(stream->poll_period_ns)); in xe_oa_poll_check_timer_cb()
316 int report_size = stream->oa_buffer.format->size; in xe_oa_append_report()
320 if ((count - *offset) < report_size) in xe_oa_append_report()
321 return -ENOSPC; in xe_oa_append_report()
325 oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; in xe_oa_append_report()
326 report_size_partial = oa_buf_end - report; in xe_oa_append_report()
330 return -EFAULT; in xe_oa_append_report()
333 if (copy_to_user(buf, stream->oa_buffer.vaddr, in xe_oa_append_report()
334 report_size - report_size_partial)) in xe_oa_append_report()
335 return -EFAULT; in xe_oa_append_report()
337 return -EFAULT; in xe_oa_append_report()
348 int report_size = stream->oa_buffer.format->size; in xe_oa_append_reports()
349 u8 *oa_buf_base = stream->oa_buffer.vaddr; in xe_oa_append_reports()
350 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); in xe_oa_append_reports()
356 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
357 head = stream->oa_buffer.head; in xe_oa_append_reports()
358 tail = stream->oa_buffer.tail; in xe_oa_append_reports()
359 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
361 xe_assert(stream->oa->xe, in xe_oa_append_reports()
362 head < stream->oa_buffer.circ_size && tail < stream->oa_buffer.circ_size); in xe_oa_append_reports()
372 if (!(stream->oa_buffer.circ_size % report_size)) { in xe_oa_append_reports()
377 u8 *oa_buf_end = stream->oa_buffer.vaddr + stream->oa_buffer.circ_size; in xe_oa_append_reports()
378 u32 part = oa_buf_end - report; in xe_oa_append_reports()
385 memset(oa_buf_base, 0, report_size - part); in xe_oa_append_reports()
391 struct xe_reg oaheadptr = __oa_regs(stream)->oa_head_ptr; in xe_oa_append_reports()
393 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
394 xe_mmio_write32(&stream->gt->mmio, oaheadptr, in xe_oa_append_reports()
396 stream->oa_buffer.head = head; in xe_oa_append_reports()
397 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_append_reports()
405 u32 gtt_offset = xe_bo_ggtt_addr(stream->oa_buffer.bo); in xe_oa_init_oa_buffer()
406 int size_exponent = __ffs(xe_bo_size(stream->oa_buffer.bo)); in xe_oa_init_oa_buffer()
408 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_init_oa_buffer()
416 size_exponent > 24 ? size_exponent - 20 : size_exponent - 17); in xe_oa_init_oa_buffer()
418 spin_lock_irqsave(&stream->oa_buffer.ptr_lock, flags); in xe_oa_init_oa_buffer()
420 xe_mmio_write32(mmio, __oa_regs(stream)->oa_status, 0); in xe_oa_init_oa_buffer()
421 xe_mmio_write32(mmio, __oa_regs(stream)->oa_head_ptr, in xe_oa_init_oa_buffer()
423 stream->oa_buffer.head = 0; in xe_oa_init_oa_buffer()
428 xe_mmio_write32(mmio, __oa_regs(stream)->oa_buffer, oa_buf); in xe_oa_init_oa_buffer()
429 xe_mmio_write32(mmio, __oa_regs(stream)->oa_tail_ptr, in xe_oa_init_oa_buffer()
433 stream->oa_buffer.tail = 0; in xe_oa_init_oa_buffer()
435 spin_unlock_irqrestore(&stream->oa_buffer.ptr_lock, flags); in xe_oa_init_oa_buffer()
438 memset(stream->oa_buffer.vaddr, 0, xe_bo_size(stream->oa_buffer.bo)); in xe_oa_init_oa_buffer()
443 return ((format->counter_select << (ffs(counter_sel_mask) - 1)) & counter_sel_mask) | in __format_to_oactrl()
444 REG_FIELD_PREP(OA_OACONTROL_REPORT_BC_MASK, format->bc_report) | in __format_to_oactrl()
445 REG_FIELD_PREP(OA_OACONTROL_COUNTER_SIZE_MASK, format->counter_size); in __format_to_oactrl()
452 if (stream->hwe->class != XE_ENGINE_CLASS_COMPUTE) in __oa_ccs_select()
455 val = REG_FIELD_PREP(OAG_OACONTROL_OA_CCS_SELECT_MASK, stream->hwe->instance); in __oa_ccs_select()
456 xe_assert(stream->oa->xe, in __oa_ccs_select()
457 REG_FIELD_GET(OAG_OACONTROL_OA_CCS_SELECT_MASK, val) == stream->hwe->instance); in __oa_ccs_select()
463 return stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG ? in __oactrl_used_bits()
469 const struct xe_oa_format *format = stream->oa_buffer.format; in xe_oa_enable()
470 const struct xe_oa_regs *regs; in xe_oa_enable() local
474 * BSpec: 46822: Bit 0. Even if stream->sample is 0, for OAR to function, the OA in xe_oa_enable()
479 regs = __oa_regs(stream); in xe_oa_enable()
480 val = __format_to_oactrl(format, regs->oa_ctrl_counter_select_mask) | in xe_oa_enable()
483 if (GRAPHICS_VER(stream->oa->xe) >= 20 && in xe_oa_enable()
484 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG) in xe_oa_enable()
487 xe_mmio_rmw32(&stream->gt->mmio, regs->oa_ctrl, __oactrl_used_bits(stream), val); in xe_oa_enable()
492 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_disable()
494 xe_mmio_rmw32(mmio, __oa_regs(stream)->oa_ctrl, __oactrl_used_bits(stream), 0); in xe_oa_disable()
495 if (xe_mmio_wait32(mmio, __oa_regs(stream)->oa_ctrl, in xe_oa_disable()
497 drm_err(&stream->oa->xe->drm, in xe_oa_disable()
500 if (GRAPHICS_VERx100(stream->oa->xe) <= 1270 && GRAPHICS_VERx100(stream->oa->xe) != 1260) { in xe_oa_disable()
504 drm_err(&stream->oa->xe->drm, in xe_oa_disable()
512 if (!stream->periodic) in xe_oa_wait_unlocked()
513 return -EINVAL; in xe_oa_wait_unlocked()
515 return wait_event_interruptible(stream->poll_wq, in xe_oa_wait_unlocked()
525 /* Only clear our bits to avoid side-effects */ in __xe_oa_read()
526 stream->oa_status = xe_mmio_rmw32(&stream->gt->mmio, __oa_regs(stream)->oa_status, in __xe_oa_read()
529 * Signal to userspace that there is non-zero OA status to read via in __xe_oa_read()
532 if (stream->oa_status & OASTATUS_RELEVANT_BITS) in __xe_oa_read()
533 return -EIO; in __xe_oa_read()
541 struct xe_oa_stream *stream = file->private_data; in xe_oa_read()
546 if (!stream->enabled || !stream->sample) in xe_oa_read()
547 return -EINVAL; in xe_oa_read()
549 if (!(file->f_flags & O_NONBLOCK)) { in xe_oa_read()
555 mutex_lock(&stream->stream_lock); in xe_oa_read()
557 mutex_unlock(&stream->stream_lock); in xe_oa_read()
561 mutex_lock(&stream->stream_lock); in xe_oa_read()
563 mutex_unlock(&stream->stream_lock); in xe_oa_read()
568 * before unblocking. The exception to this is if __xe_oa_read returns -ENOSPC, in xe_oa_read()
572 * Also in case of -EIO, we have already waited for data before returning in xe_oa_read()
573 * -EIO, so need to wait again in xe_oa_read()
575 if (ret != -ENOSPC && ret != -EIO) in xe_oa_read()
576 stream->pollin = false; in xe_oa_read()
578 /* Possible values for ret are 0, -EFAULT, -ENOSPC, -EIO, -EINVAL, ... */ in xe_oa_read()
579 return offset ?: (ret ?: -EAGAIN); in xe_oa_read()
587 poll_wait(file, &stream->poll_wq, wait); in xe_oa_poll_locked()
595 if (stream->pollin) in xe_oa_poll_locked()
603 struct xe_oa_stream *stream = file->private_data; in xe_oa_poll()
606 mutex_lock(&stream->stream_lock); in xe_oa_poll()
608 mutex_unlock(&stream->stream_lock); in xe_oa_poll()
615 if (q->vm) { in xe_oa_lock_vma()
616 down_read(&q->vm->lock); in xe_oa_lock_vma()
617 xe_vm_lock(q->vm, false); in xe_oa_lock_vma()
623 if (q->vm) { in xe_oa_unlock_vma()
624 xe_vm_unlock(q->vm); in xe_oa_unlock_vma()
625 up_read(&q->vm->lock); in xe_oa_unlock_vma()
632 struct xe_exec_queue *q = stream->exec_q ?: stream->k_exec_q; in xe_oa_submit_bb()
644 job->ggtt = true; in xe_oa_submit_bb()
647 for (int i = 0; i < stream->num_syncs && !err; i++) in xe_oa_submit_bb()
648 err = xe_sync_entry_add_deps(&stream->syncs[i], job); in xe_oa_submit_bb()
650 drm_dbg(&stream->oa->xe->drm, "xe_sync_entry_add_deps err %d\n", err); in xe_oa_submit_bb()
656 fence = dma_fence_get(&job->drm.s_fence->finished); in xe_oa_submit_bb()
677 u32 n_lri = min_t(u32, n_regs - i, in write_cs_mi_lri()
680 bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM | MI_LRI_NUM_REGS(n_lri); in write_cs_mi_lri()
682 bb->cs[bb->len++] = reg_data[i].addr.addr; in write_cs_mi_lri()
683 bb->cs[bb->len++] = reg_data[i].value; in write_cs_mi_lri()
701 xe_bo_unpin_map_no_vm(stream->oa_buffer.bo); in xe_oa_free_oa_buffer()
708 xe_oa_config_put(stream->oa_config); in xe_oa_free_configs()
709 llist_for_each_entry_safe(oa_bo, tmp, stream->oa_config_bos.first, node) in xe_oa_free_configs()
710 free_oa_config_bo(oa_bo, stream->last_fence); in xe_oa_free_configs()
711 dma_fence_put(stream->last_fence); in xe_oa_free_configs()
720 bb = xe_bb_new(stream->gt, 2 * count + 1, false); in xe_oa_load_with_lri()
745 const struct xe_oa_format *format = stream->oa_buffer.format; in xe_oa_configure_oar_context()
751 OACTXCONTROL(stream->hwe->mmio_base), in xe_oa_configure_oar_context()
759 RING_CONTEXT_CONTROL(stream->hwe->mmio_base), in xe_oa_configure_oar_context()
770 const struct xe_oa_format *format = stream->oa_buffer.format; in xe_oa_configure_oac_context()
775 OACTXCONTROL(stream->hwe->mmio_base), in xe_oa_configure_oac_context()
783 RING_CONTEXT_CONTROL(stream->hwe->mmio_base), in xe_oa_configure_oac_context()
791 xe_mmio_write32(&stream->gt->mmio, __oa_regs(stream)->oa_ctrl, in xe_oa_configure_oac_context()
799 switch (stream->hwe->class) { in xe_oa_configure_oa_context()
815 enable && stream && stream->sample ? in oag_configure_mmio_trigger()
821 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_disable_metric_set()
825 if (XE_WA(stream->gt, 1508761755)) { in xe_oa_disable_metric_set()
826 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, in xe_oa_disable_metric_set()
828 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, in xe_oa_disable_metric_set()
832 xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug, in xe_oa_disable_metric_set()
836 if (stream->exec_q) in xe_oa_disable_metric_set()
843 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); in xe_oa_disable_metric_set()
848 if ((stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM || in xe_oa_disable_metric_set()
849 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM_SAG) && in xe_oa_disable_metric_set()
850 GRAPHICS_VER(stream->oa->xe) >= 30) in xe_oa_disable_metric_set()
856 struct xe_oa_unit *u = stream->oa_unit; in xe_oa_stream_destroy()
857 struct xe_gt *gt = stream->hwe->gt; in xe_oa_stream_destroy()
859 if (WARN_ON(stream != u->exclusive_stream)) in xe_oa_stream_destroy()
862 WRITE_ONCE(u->exclusive_stream, NULL); in xe_oa_stream_destroy()
864 mutex_destroy(&stream->stream_lock); in xe_oa_stream_destroy()
867 xe_exec_queue_put(stream->k_exec_q); in xe_oa_stream_destroy()
872 xe_pm_runtime_put(stream->oa->xe); in xe_oa_stream_destroy()
875 if (stream->override_gucrc) in xe_oa_stream_destroy()
876 xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(>->uc.guc.pc)); in xe_oa_stream_destroy()
879 xe_file_put(stream->xef); in xe_oa_stream_destroy()
886 bo = xe_bo_create_pin_map(stream->oa->xe, stream->gt->tile, NULL, in xe_oa_alloc_oa_buffer()
892 stream->oa_buffer.bo = bo; in xe_oa_alloc_oa_buffer()
894 xe_assert(stream->oa->xe, bo->vmap.is_iomem == 0); in xe_oa_alloc_oa_buffer()
895 stream->oa_buffer.vaddr = bo->vmap.vaddr; in xe_oa_alloc_oa_buffer()
908 return ERR_PTR(-ENOMEM); in __xe_oa_alloc_config_buffer()
910 config_length = num_lri_dwords(oa_config->regs_len); in __xe_oa_alloc_config_buffer()
913 bb = xe_bb_new(stream->gt, config_length, false); in __xe_oa_alloc_config_buffer()
917 write_cs_mi_lri(bb, oa_config->regs, oa_config->regs_len); in __xe_oa_alloc_config_buffer()
919 oa_bo->bb = bb; in __xe_oa_alloc_config_buffer()
920 oa_bo->oa_config = xe_oa_config_get(oa_config); in __xe_oa_alloc_config_buffer()
921 llist_add(&oa_bo->node, &stream->oa_config_bos); in __xe_oa_alloc_config_buffer()
935 llist_for_each_entry(oa_bo, stream->oa_config_bos.first, node) { in xe_oa_alloc_config_buffer()
936 if (oa_bo->oa_config == oa_config && in xe_oa_alloc_config_buffer()
937 memcmp(oa_bo->oa_config->uuid, oa_config->uuid, in xe_oa_alloc_config_buffer()
938 sizeof(oa_config->uuid)) == 0) in xe_oa_alloc_config_buffer()
949 dma_fence_put(stream->last_fence); in xe_oa_update_last_fence()
950 stream->last_fence = dma_fence_get(fence); in xe_oa_update_last_fence()
958 dma_fence_signal(&ofence->base); in xe_oa_fence_work_fn()
959 dma_fence_put(&ofence->base); in xe_oa_fence_work_fn()
969 INIT_DELAYED_WORK(&ofence->work, xe_oa_fence_work_fn); in xe_oa_config_cb()
970 queue_delayed_work(system_unbound_wq, &ofence->work, in xe_oa_config_cb()
1000 err = -ENOMEM; in xe_oa_emit_oa_config()
1011 fence = xe_oa_submit_bb(stream, XE_OA_SUBMIT_ADD_DEPS, oa_bo->bb); in xe_oa_emit_oa_config()
1018 spin_lock_init(&ofence->lock); in xe_oa_emit_oa_config()
1019 dma_fence_init(&ofence->base, &xe_oa_fence_ops, &ofence->lock, 0, 0); in xe_oa_emit_oa_config()
1021 for (i = 0; i < stream->num_syncs; i++) { in xe_oa_emit_oa_config()
1022 if (stream->syncs[i].flags & DRM_XE_SYNC_FLAG_SIGNAL) in xe_oa_emit_oa_config()
1024 xe_sync_entry_signal(&stream->syncs[i], &ofence->base); in xe_oa_emit_oa_config()
1029 dma_fence_get(&ofence->base); in xe_oa_emit_oa_config()
1034 /* Add job fence callback to schedule work to signal ofence->base */ in xe_oa_emit_oa_config()
1035 err = dma_fence_add_callback(fence, &ofence->cb, xe_oa_config_cb); in xe_oa_emit_oa_config()
1036 xe_gt_assert(stream->gt, !err || err == -ENOENT); in xe_oa_emit_oa_config()
1037 if (err == -ENOENT) in xe_oa_emit_oa_config()
1038 xe_oa_config_cb(fence, &ofence->cb); in xe_oa_emit_oa_config()
1042 dma_fence_wait(&ofence->base, false); in xe_oa_emit_oa_config()
1043 dma_fence_put(&ofence->base); in xe_oa_emit_oa_config()
1047 for (i = 0; i < stream->num_syncs; i++) in xe_oa_emit_oa_config()
1048 xe_sync_entry_cleanup(&stream->syncs[i]); in xe_oa_emit_oa_config()
1049 kfree(stream->syncs); in xe_oa_emit_oa_config()
1061 stream->sample ? in oag_report_ctx_switches()
1068 xe_bo_size(stream->oa_buffer.bo) > SZ_16M ? in oag_buf_size_select()
1074 struct xe_mmio *mmio = &stream->gt->mmio; in xe_oa_enable_metric_set()
1082 if (XE_WA(stream->gt, 1508761755)) { in xe_oa_enable_metric_set()
1083 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN, in xe_oa_enable_metric_set()
1085 xe_gt_mcr_multicast_write(stream->gt, ROW_CHICKEN2, in xe_oa_enable_metric_set()
1093 if (GRAPHICS_VER(stream->oa->xe) >= 20) in xe_oa_enable_metric_set()
1100 xe_mmio_write32(mmio, __oa_regs(stream)->oa_debug, in xe_oa_enable_metric_set()
1106 xe_mmio_write32(mmio, __oa_regs(stream)->oa_ctx_ctrl, stream->periodic ? in xe_oa_enable_metric_set()
1110 stream->period_exponent)) : 0); in xe_oa_enable_metric_set()
1118 (HAS_OA_BPC_REPORTING(stream->oa->xe) ? SQCNT1_OABPC : 0); in xe_oa_enable_metric_set()
1121 if ((stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM || in xe_oa_enable_metric_set()
1122 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAM_SAG) && in xe_oa_enable_metric_set()
1123 GRAPHICS_VER(stream->oa->xe) >= 30) in xe_oa_enable_metric_set()
1127 if (stream->exec_q) { in xe_oa_enable_metric_set()
1133 return xe_oa_emit_oa_config(stream, stream->oa_config); in xe_oa_enable_metric_set()
1144 for_each_set_bit(idx, oa->format_mask, __XE_OA_FORMAT_MAX) { in decode_oa_format()
1145 const struct xe_oa_format *f = &oa->oa_formats[idx]; in decode_oa_format()
1147 if (counter_size == f->counter_size && bc_report == f->bc_report && in decode_oa_format()
1148 type == f->type && counter_sel == f->counter_select) { in decode_oa_format()
1154 return -EINVAL; in decode_oa_format()
1162 for_each_gt(gt, oa->xe, gt_id) { in xe_oa_lookup_oa_unit()
1163 for (i = 0; i < gt->oa.num_oa_units; i++) { in xe_oa_lookup_oa_unit()
1164 struct xe_oa_unit *u = >->oa.oa_unit[i]; in xe_oa_lookup_oa_unit()
1166 if (u->oa_unit_id == oa_unit_id) in xe_oa_lookup_oa_unit()
1177 param->oa_unit = xe_oa_lookup_oa_unit(oa, value); in xe_oa_set_prop_oa_unit_id()
1178 if (!param->oa_unit) { in xe_oa_set_prop_oa_unit_id()
1179 drm_dbg(&oa->xe->drm, "OA unit ID out of range %lld\n", value); in xe_oa_set_prop_oa_unit_id()
1180 return -EINVAL; in xe_oa_set_prop_oa_unit_id()
1188 param->sample = value; in xe_oa_set_prop_sample_oa()
1195 param->metric_set = value; in xe_oa_set_prop_metric_set()
1202 int ret = decode_oa_format(oa, value, ¶m->oa_format); in xe_oa_set_prop_oa_format()
1205 drm_dbg(&oa->xe->drm, "Unsupported OA report format %#llx\n", value); in xe_oa_set_prop_oa_format()
1217 drm_dbg(&oa->xe->drm, "OA timer exponent too high (> %u)\n", OA_EXPONENT_MAX); in xe_oa_set_prop_oa_exponent()
1218 return -EINVAL; in xe_oa_set_prop_oa_exponent()
1220 param->period_exponent = value; in xe_oa_set_prop_oa_exponent()
1227 param->disabled = value; in xe_oa_set_prop_disabled()
1234 param->exec_queue_id = value; in xe_oa_set_prop_exec_queue_id()
1241 param->engine_instance = value; in xe_oa_set_prop_engine_instance()
1248 param->no_preempt = value; in xe_oa_set_no_preempt()
1255 param->num_syncs = value; in xe_oa_set_prop_num_syncs()
1262 param->syncs_user = u64_to_user_ptr(value); in xe_oa_set_prop_syncs_user()
1270 drm_dbg(&oa->xe->drm, "OA buffer size invalid %llu\n", value); in xe_oa_set_prop_oa_buffer_size()
1271 return -EINVAL; in xe_oa_set_prop_oa_buffer_size()
1273 param->oa_buffer_size = value; in xe_oa_set_prop_oa_buffer_size()
1281 drm_dbg(&oa->xe->drm, "wait_num_reports %llu\n", value); in xe_oa_set_prop_wait_num_reports()
1282 return -EINVAL; in xe_oa_set_prop_wait_num_reports()
1284 param->wait_num_reports = value; in xe_oa_set_prop_wait_num_reports()
1291 return -EINVAL; in xe_oa_set_prop_ret_inval()
1332 struct drm_xe_ext_set_property ext; in xe_oa_user_ext_set_property() local
1336 err = copy_from_user(&ext, address, sizeof(ext)); in xe_oa_user_ext_set_property()
1337 if (XE_IOCTL_DBG(oa->xe, err)) in xe_oa_user_ext_set_property()
1338 return -EFAULT; in xe_oa_user_ext_set_property()
1343 if (XE_IOCTL_DBG(oa->xe, ext.property >= ARRAY_SIZE(xe_oa_set_property_funcs_open)) || in xe_oa_user_ext_set_property()
1344 XE_IOCTL_DBG(oa->xe, ext.pad)) in xe_oa_user_ext_set_property()
1345 return -EINVAL; in xe_oa_user_ext_set_property()
1347 idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_oa_set_property_funcs_open)); in xe_oa_user_ext_set_property()
1350 return xe_oa_set_property_funcs_config[idx](oa, ext.value, param); in xe_oa_user_ext_set_property()
1352 return xe_oa_set_property_funcs_open[idx](oa, ext.value, param); in xe_oa_user_ext_set_property()
1366 struct drm_xe_user_extension ext; in xe_oa_user_extensions() local
1370 if (XE_IOCTL_DBG(oa->xe, ext_number >= MAX_USER_EXTENSIONS)) in xe_oa_user_extensions()
1371 return -E2BIG; in xe_oa_user_extensions()
1373 err = copy_from_user(&ext, address, sizeof(ext)); in xe_oa_user_extensions()
1374 if (XE_IOCTL_DBG(oa->xe, err)) in xe_oa_user_extensions()
1375 return -EFAULT; in xe_oa_user_extensions()
1377 if (XE_IOCTL_DBG(oa->xe, ext.pad) || in xe_oa_user_extensions()
1378 XE_IOCTL_DBG(oa->xe, ext.name >= ARRAY_SIZE(xe_oa_user_extension_funcs))) in xe_oa_user_extensions()
1379 return -EINVAL; in xe_oa_user_extensions()
1381 idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_oa_user_extension_funcs)); in xe_oa_user_extensions()
1383 if (XE_IOCTL_DBG(oa->xe, err)) in xe_oa_user_extensions()
1386 if (ext.next_extension) in xe_oa_user_extensions()
1387 return xe_oa_user_extensions(oa, from, ext.next_extension, ++ext_number, param); in xe_oa_user_extensions()
1396 if (param->num_syncs && !param->syncs_user) { in xe_oa_parse_syncs()
1397 drm_dbg(&oa->xe->drm, "num_syncs specified without sync array\n"); in xe_oa_parse_syncs()
1398 ret = -EINVAL; in xe_oa_parse_syncs()
1402 if (param->num_syncs) { in xe_oa_parse_syncs()
1403 param->syncs = kcalloc(param->num_syncs, sizeof(*param->syncs), GFP_KERNEL); in xe_oa_parse_syncs()
1404 if (!param->syncs) { in xe_oa_parse_syncs()
1405 ret = -ENOMEM; in xe_oa_parse_syncs()
1410 for (num_syncs = 0; num_syncs < param->num_syncs; num_syncs++) { in xe_oa_parse_syncs()
1411 ret = xe_sync_entry_parse(oa->xe, param->xef, ¶m->syncs[num_syncs], in xe_oa_parse_syncs()
1412 ¶m->syncs_user[num_syncs], 0); in xe_oa_parse_syncs()
1416 if (xe_sync_is_ufence(¶m->syncs[num_syncs])) in xe_oa_parse_syncs()
1420 if (XE_IOCTL_DBG(oa->xe, num_ufence > 1)) { in xe_oa_parse_syncs()
1421 ret = -EINVAL; in xe_oa_parse_syncs()
1428 while (num_syncs--) in xe_oa_parse_syncs()
1429 xe_sync_entry_cleanup(¶m->syncs[num_syncs]); in xe_oa_parse_syncs()
1430 kfree(param->syncs); in xe_oa_parse_syncs()
1437 stream->pollin = false; in xe_oa_stream_enable()
1441 if (stream->sample) in xe_oa_stream_enable()
1442 hrtimer_start(&stream->poll_check_timer, in xe_oa_stream_enable()
1443 ns_to_ktime(stream->poll_period_ns), in xe_oa_stream_enable()
1451 if (stream->sample) in xe_oa_stream_disable()
1452 hrtimer_cancel(&stream->poll_check_timer); in xe_oa_stream_disable()
1457 struct xe_exec_queue *q = stream->exec_q; in xe_oa_enable_preempt_timeslice()
1461 ret1 = q->ops->set_timeslice(q, stream->hwe->eclass->sched_props.timeslice_us); in xe_oa_enable_preempt_timeslice()
1462 ret2 = q->ops->set_preempt_timeout(q, stream->hwe->eclass->sched_props.preempt_timeout_us); in xe_oa_enable_preempt_timeslice()
1467 drm_dbg(&stream->oa->xe->drm, "%s failed ret1 %d ret2 %d\n", __func__, ret1, ret2); in xe_oa_enable_preempt_timeslice()
1473 struct xe_exec_queue *q = stream->exec_q; in xe_oa_disable_preempt_timeslice()
1477 ret = q->ops->set_timeslice(q, 0); in xe_oa_disable_preempt_timeslice()
1481 ret = q->ops->set_preempt_timeout(q, 0); in xe_oa_disable_preempt_timeslice()
1488 drm_dbg(&stream->oa->xe->drm, "%s failed %d\n", __func__, ret); in xe_oa_disable_preempt_timeslice()
1494 if (stream->enabled) in xe_oa_enable_locked()
1497 if (stream->no_preempt) { in xe_oa_enable_locked()
1506 stream->enabled = true; in xe_oa_enable_locked()
1514 if (!stream->enabled) in xe_oa_disable_locked()
1519 if (stream->no_preempt) in xe_oa_disable_locked()
1522 stream->enabled = false; in xe_oa_disable_locked()
1529 long ret = stream->oa_config->id; in xe_oa_config_locked()
1533 err = xe_oa_user_extensions(stream->oa, XE_OA_USER_EXTN_FROM_CONFIG, arg, 0, ¶m); in xe_oa_config_locked()
1537 config = xe_oa_get_oa_config(stream->oa, param.metric_set); in xe_oa_config_locked()
1539 return -ENODEV; in xe_oa_config_locked()
1541 param.xef = stream->xef; in xe_oa_config_locked()
1542 err = xe_oa_parse_syncs(stream->oa, ¶m); in xe_oa_config_locked()
1546 stream->num_syncs = param.num_syncs; in xe_oa_config_locked()
1547 stream->syncs = param.syncs; in xe_oa_config_locked()
1551 config = xchg(&stream->oa_config, config); in xe_oa_config_locked()
1552 drm_dbg(&stream->oa->xe->drm, "changed to oa config uuid=%s\n", in xe_oa_config_locked()
1553 stream->oa_config->uuid); in xe_oa_config_locked()
1568 if (stream->oa_status & OASTATUS_REPORT_LOST) in xe_oa_status_locked()
1570 if (stream->oa_status & OASTATUS_BUFFER_OVERFLOW) in xe_oa_status_locked()
1572 if (stream->oa_status & OASTATUS_COUNTER_OVERFLOW) in xe_oa_status_locked()
1574 if (stream->oa_status & OASTATUS_MMIO_TRG_Q_FULL) in xe_oa_status_locked()
1578 return -EFAULT; in xe_oa_status_locked()
1585 struct drm_xe_oa_stream_info info = { .oa_buf_size = xe_bo_size(stream->oa_buffer.bo), }; in xe_oa_info_locked()
1589 return -EFAULT; in xe_oa_info_locked()
1611 return -EINVAL; in xe_oa_ioctl_locked()
1618 struct xe_oa_stream *stream = file->private_data; in xe_oa_ioctl()
1621 mutex_lock(&stream->stream_lock); in xe_oa_ioctl()
1623 mutex_unlock(&stream->stream_lock); in xe_oa_ioctl()
1630 if (stream->enabled) in xe_oa_destroy_locked()
1635 if (stream->exec_q) in xe_oa_destroy_locked()
1636 xe_exec_queue_put(stream->exec_q); in xe_oa_destroy_locked()
1643 struct xe_oa_stream *stream = file->private_data; in xe_oa_release()
1644 struct xe_gt *gt = stream->gt; in xe_oa_release()
1647 mutex_lock(>->oa.gt_lock); in xe_oa_release()
1649 mutex_unlock(>->oa.gt_lock); in xe_oa_release()
1653 drm_dev_put(>_to_xe(gt)->drm); in xe_oa_release()
1660 struct xe_oa_stream *stream = file->private_data; in xe_oa_mmap()
1661 struct xe_bo *bo = stream->oa_buffer.bo; in xe_oa_mmap()
1662 unsigned long start = vma->vm_start; in xe_oa_mmap()
1666 drm_dbg(&stream->oa->xe->drm, "Insufficient privilege to map OA buffer\n"); in xe_oa_mmap()
1667 return -EACCES; in xe_oa_mmap()
1671 if (vma->vm_end - vma->vm_start != xe_bo_size(stream->oa_buffer.bo)) { in xe_oa_mmap()
1672 drm_dbg(&stream->oa->xe->drm, "Wrong mmap size, must be OA buffer size\n"); in xe_oa_mmap()
1673 return -EINVAL; in xe_oa_mmap()
1680 if (vma->vm_flags & (VM_WRITE | VM_EXEC | VM_SHARED | VM_MAYSHARE)) { in xe_oa_mmap()
1681 drm_dbg(&stream->oa->xe->drm, "mmap must be read only\n"); in xe_oa_mmap()
1682 return -EINVAL; in xe_oa_mmap()
1687 xe_assert(stream->oa->xe, bo->ttm.ttm->num_pages == vma_pages(vma)); in xe_oa_mmap()
1688 for (i = 0; i < bo->ttm.ttm->num_pages; i++) { in xe_oa_mmap()
1689 ret = remap_pfn_range(vma, start, page_to_pfn(bo->ttm.ttm->pages[i]), in xe_oa_mmap()
1690 PAGE_SIZE, vma->vm_page_prot); in xe_oa_mmap()
1712 struct xe_gt *gt = param->hwe->gt; in xe_oa_stream_init()
1716 stream->exec_q = param->exec_q; in xe_oa_stream_init()
1717 stream->poll_period_ns = DEFAULT_POLL_PERIOD_NS; in xe_oa_stream_init()
1718 stream->oa_unit = param->oa_unit; in xe_oa_stream_init()
1719 stream->hwe = param->hwe; in xe_oa_stream_init()
1720 stream->gt = stream->hwe->gt; in xe_oa_stream_init()
1721 stream->oa_buffer.format = &stream->oa->oa_formats[param->oa_format]; in xe_oa_stream_init()
1723 stream->sample = param->sample; in xe_oa_stream_init()
1724 stream->periodic = param->period_exponent >= 0; in xe_oa_stream_init()
1725 stream->period_exponent = param->period_exponent; in xe_oa_stream_init()
1726 stream->no_preempt = param->no_preempt; in xe_oa_stream_init()
1727 stream->wait_num_reports = param->wait_num_reports; in xe_oa_stream_init()
1729 stream->xef = xe_file_get(param->xef); in xe_oa_stream_init()
1730 stream->num_syncs = param->num_syncs; in xe_oa_stream_init()
1731 stream->syncs = param->syncs; in xe_oa_stream_init()
1735 * of buffer, making the OA buffer effectively a non-power-of-2 size circular in xe_oa_stream_init()
1738 if (GRAPHICS_VER(stream->oa->xe) >= 20 && in xe_oa_stream_init()
1739 stream->oa_unit->type == DRM_XE_OA_UNIT_TYPE_OAG && stream->sample) in xe_oa_stream_init()
1740 stream->oa_buffer.circ_size = in xe_oa_stream_init()
1741 param->oa_buffer_size - in xe_oa_stream_init()
1742 param->oa_buffer_size % stream->oa_buffer.format->size; in xe_oa_stream_init()
1744 stream->oa_buffer.circ_size = param->oa_buffer_size; in xe_oa_stream_init()
1746 stream->oa_config = xe_oa_get_oa_config(stream->oa, param->metric_set); in xe_oa_stream_init()
1747 if (!stream->oa_config) { in xe_oa_stream_init()
1748 drm_dbg(&stream->oa->xe->drm, "Invalid OA config id=%i\n", param->metric_set); in xe_oa_stream_init()
1749 ret = -EINVAL; in xe_oa_stream_init()
1757 if (XE_WA(stream->gt, 1509372804)) { in xe_oa_stream_init()
1758 ret = xe_guc_pc_override_gucrc_mode(>->uc.guc.pc, in xe_oa_stream_init()
1763 stream->override_gucrc = true; in xe_oa_stream_init()
1767 xe_pm_runtime_get(stream->oa->xe); in xe_oa_stream_init()
1770 ret = -ETIMEDOUT; in xe_oa_stream_init()
1774 ret = xe_oa_alloc_oa_buffer(stream, param->oa_buffer_size); in xe_oa_stream_init()
1778 stream->k_exec_q = xe_exec_queue_create(stream->oa->xe, NULL, in xe_oa_stream_init()
1779 BIT(stream->hwe->logical_instance), 1, in xe_oa_stream_init()
1780 stream->hwe, EXEC_QUEUE_FLAG_KERNEL, 0); in xe_oa_stream_init()
1781 if (IS_ERR(stream->k_exec_q)) { in xe_oa_stream_init()
1782 ret = PTR_ERR(stream->k_exec_q); in xe_oa_stream_init()
1783 drm_err(&stream->oa->xe->drm, "gt%d, hwe %s, xe_exec_queue_create failed=%d", in xe_oa_stream_init()
1784 stream->gt->info.id, stream->hwe->name, ret); in xe_oa_stream_init()
1790 drm_dbg(&stream->oa->xe->drm, "Unable to enable metric set\n"); in xe_oa_stream_init()
1794 drm_dbg(&stream->oa->xe->drm, "opening stream oa config uuid=%s\n", in xe_oa_stream_init()
1795 stream->oa_config->uuid); in xe_oa_stream_init()
1797 WRITE_ONCE(stream->oa_unit->exclusive_stream, stream); in xe_oa_stream_init()
1799 hrtimer_setup(&stream->poll_check_timer, xe_oa_poll_check_timer_cb, CLOCK_MONOTONIC, in xe_oa_stream_init()
1801 init_waitqueue_head(&stream->poll_wq); in xe_oa_stream_init()
1803 spin_lock_init(&stream->oa_buffer.ptr_lock); in xe_oa_stream_init()
1804 mutex_init(&stream->stream_lock); in xe_oa_stream_init()
1810 xe_exec_queue_put(stream->k_exec_q); in xe_oa_stream_init()
1815 xe_pm_runtime_put(stream->oa->xe); in xe_oa_stream_init()
1816 if (stream->override_gucrc) in xe_oa_stream_init()
1817 xe_gt_WARN_ON(gt, xe_guc_pc_unset_gucrc_mode(>->uc.guc.pc)); in xe_oa_stream_init()
1821 xe_file_put(stream->xef); in xe_oa_stream_init()
1833 if (param->oa_unit->exclusive_stream) { in xe_oa_stream_open_ioctl_locked()
1834 drm_dbg(&oa->xe->drm, "OA unit already in use\n"); in xe_oa_stream_open_ioctl_locked()
1835 ret = -EBUSY; in xe_oa_stream_open_ioctl_locked()
1841 ret = -ENOMEM; in xe_oa_stream_open_ioctl_locked()
1845 stream->oa = oa; in xe_oa_stream_open_ioctl_locked()
1850 if (!param->disabled) { in xe_oa_stream_open_ioctl_locked()
1863 drm_dev_get(&stream->oa->xe->drm); in xe_oa_stream_open_ioctl_locked()
1867 if (!param->disabled) in xe_oa_stream_open_ioctl_locked()
1878 * xe_oa_timestamp_frequency - Return OA timestamp frequency
1891 reg = xe_mmio_read32(>->mmio, RPM_CONFIG0); in xe_oa_timestamp_frequency()
1895 return gt->info.reference_clock << (3 - shift); in xe_oa_timestamp_frequency()
1897 return gt->info.reference_clock; in xe_oa_timestamp_frequency()
1906 return div_u64(nom + den - 1, den); in oa_exponent_to_ns()
1911 switch (param->oa_unit->type) { in oa_unit_supports_oa_format()
1924 * xe_oa_unit_id - Return OA unit ID for a hardware engine
1931 return hwe->oa_unit && hwe->oa_unit->num_engines ? in xe_oa_unit_id()
1932 hwe->oa_unit->oa_unit_id : U16_MAX; in xe_oa_unit_id()
1943 if (!param->oa_unit) in xe_oa_assign_hwe()
1944 param->oa_unit = &xe_root_mmio_gt(oa->xe)->oa.oa_unit[0]; in xe_oa_assign_hwe()
1947 if (param->exec_q) { in xe_oa_assign_hwe()
1948 param->hwe = xe_gt_hw_engine(param->exec_q->gt, param->exec_q->class, in xe_oa_assign_hwe()
1949 param->engine_instance, true); in xe_oa_assign_hwe()
1950 if (!param->hwe || param->hwe->oa_unit != param->oa_unit) in xe_oa_assign_hwe()
1956 for_each_hw_engine(hwe, param->oa_unit->gt, id) { in xe_oa_assign_hwe()
1957 if (hwe->oa_unit == param->oa_unit) { in xe_oa_assign_hwe()
1958 param->hwe = hwe; in xe_oa_assign_hwe()
1964 for_each_hw_engine(hwe, param->oa_unit->gt, id) { in xe_oa_assign_hwe()
1965 if (!hwe->oa_unit) in xe_oa_assign_hwe()
1968 param->hwe = hwe; in xe_oa_assign_hwe()
1972 drm_dbg(&oa->xe->drm, "Unable to find hwe (%d, %d) for OA unit ID %d\n", in xe_oa_assign_hwe()
1973 param->exec_q ? param->exec_q->class : -1, in xe_oa_assign_hwe()
1974 param->engine_instance, param->oa_unit->oa_unit_id); in xe_oa_assign_hwe()
1975 ret = -EINVAL; in xe_oa_assign_hwe()
1981 * xe_oa_stream_open_ioctl - Opens an OA stream
1993 struct xe_oa *oa = &xe->oa; in xe_oa_stream_open_ioctl()
2000 if (!oa->xe) { in xe_oa_stream_open_ioctl()
2001 drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); in xe_oa_stream_open_ioctl()
2002 return -ENODEV; in xe_oa_stream_open_ioctl()
2006 param.period_exponent = -1; in xe_oa_stream_open_ioctl()
2013 if (XE_IOCTL_DBG(oa->xe, !param.exec_q)) in xe_oa_stream_open_ioctl()
2014 return -ENOENT; in xe_oa_stream_open_ioctl()
2016 if (XE_IOCTL_DBG(oa->xe, param.exec_q->width > 1)) in xe_oa_stream_open_ioctl()
2017 return -EOPNOTSUPP; in xe_oa_stream_open_ioctl()
2029 drm_dbg(&oa->xe->drm, "Preemption disable without exec_q!\n"); in xe_oa_stream_open_ioctl()
2030 ret = -EINVAL; in xe_oa_stream_open_ioctl()
2037 drm_dbg(&oa->xe->drm, "Insufficient privileges to open xe OA stream\n"); in xe_oa_stream_open_ioctl()
2038 ret = -EACCES; in xe_oa_stream_open_ioctl()
2043 drm_dbg(&oa->xe->drm, "Only OA report sampling supported\n"); in xe_oa_stream_open_ioctl()
2044 ret = -EINVAL; in xe_oa_stream_open_ioctl()
2052 f = &oa->oa_formats[param.oa_format]; in xe_oa_stream_open_ioctl()
2053 if (!param.oa_format || !f->size || in xe_oa_stream_open_ioctl()
2054 !oa_unit_supports_oa_format(¶m, f->type)) { in xe_oa_stream_open_ioctl()
2055 drm_dbg(&oa->xe->drm, "Invalid OA format %d type %d size %d for class %d\n", in xe_oa_stream_open_ioctl()
2056 param.oa_format, f->type, f->size, param.hwe->class); in xe_oa_stream_open_ioctl()
2057 ret = -EINVAL; in xe_oa_stream_open_ioctl()
2066 drm_dbg(&oa->xe->drm, "OA_EXPONENT specified without SAMPLE_OA\n"); in xe_oa_stream_open_ioctl()
2067 ret = -EINVAL; in xe_oa_stream_open_ioctl()
2070 oa_period = oa_exponent_to_ns(param.hwe->gt, param.period_exponent); in xe_oa_stream_open_ioctl()
2072 drm_dbg(&oa->xe->drm, "Using periodic sampling freq %lld Hz\n", oa_freq_hz); in xe_oa_stream_open_ioctl()
2080 if (param.wait_num_reports > param.oa_buffer_size / f->size) { in xe_oa_stream_open_ioctl()
2081 drm_dbg(&oa->xe->drm, "wait_num_reports %d\n", param.wait_num_reports); in xe_oa_stream_open_ioctl()
2082 ret = -EINVAL; in xe_oa_stream_open_ioctl()
2090 mutex_lock(¶m.hwe->gt->oa.gt_lock); in xe_oa_stream_open_ioctl()
2092 mutex_unlock(¶m.hwe->gt->oa.gt_lock); in xe_oa_stream_open_ioctl()
2099 while (param.num_syncs--) in xe_oa_stream_open_ioctl()
2130 while (table->start && table->end) { in xe_oa_reg_in_range_table()
2131 if (addr >= table->start && addr <= table->end) in xe_oa_reg_in_range_table()
2142 { .start = 0xdd00, .end = 0xdd48 }, /* OAG_LCE0_0 - OAA_LENABLE_REG */
2148 { .start = 0xd900, .end = 0xd91c }, /* OAG_OASTARTTRIG[1-8] */
2149 { .start = 0xd920, .end = 0xd93c }, /* OAG_OAREPORTTRIG1[1-8] */
2150 { .start = 0xd940, .end = 0xd97c }, /* OAG_CEC[0-7][0-1] */
2151 { .start = 0xdc00, .end = 0xdc3c }, /* OAG_SCEC[0-7][0-1] */
2158 { .start = 0x393000, .end = 0x39301c }, /* OAM_STARTTRIG1[1-8] */
2159 { .start = 0x393020, .end = 0x39303c }, /* OAM_REPORTTRIG1[1-8] */
2160 { .start = 0x393040, .end = 0x39307c }, /* OAM_CEC[0-7][0-1] */
2161 { .start = 0x393200, .end = 0x39323C }, /* MPES[0-7] */
2166 { .start = 0x393200, .end = 0x39323C }, /* MPES_0_MPES_SAG - MPES_7_UPPER_MPES_SAG */
2167 { .start = 0x394200, .end = 0x39423C }, /* MPES_0_MPES_SCMI0 - MPES_7_UPPER_MPES_SCMI0 */
2168 { .start = 0x394A00, .end = 0x394A3C }, /* MPES_0_MPES_SCMI1 - MPES_7_UPPER_MPES_SCMI1 */
2177 (GRAPHICS_VER(oa->xe) >= 20 && in xe_oa_is_valid_b_counter_addr()
2182 { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */
2183 { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */
2191 { .start = 0x0d00, .end = 0x0d04 }, /* RPM_CONFIG[0-1] */
2192 { .start = 0x0d0c, .end = 0x0d2c }, /* NOA_CONFIG[0-8] */
2207 { .start = 0x13000, .end = 0x137FC }, /* PES_0_PESL0 - PES_63_UPPER_PESL3 */
2213 if (GRAPHICS_VER(oa->xe) >= 20) in xe_oa_is_valid_mux_addr()
2215 else if (GRAPHICS_VERx100(oa->xe) >= 1270) in xe_oa_is_valid_mux_addr()
2230 u32 __user *regs, u32 n_regs) in xe_oa_alloc_regs() argument
2238 return ERR_PTR(-ENOMEM); in xe_oa_alloc_regs()
2243 err = get_user(addr, regs); in xe_oa_alloc_regs()
2248 drm_dbg(&oa->xe->drm, "Invalid oa_reg address: %X\n", addr); in xe_oa_alloc_regs()
2249 err = -EINVAL; in xe_oa_alloc_regs()
2253 err = get_user(value, regs + 1); in xe_oa_alloc_regs()
2260 regs += 2; in xe_oa_alloc_regs()
2278 return sysfs_emit(buf, "%d\n", oa_config->id); in show_dynamic_id()
2284 sysfs_attr_init(&oa_config->sysfs_metric_id.attr); in create_dynamic_oa_sysfs_entry()
2285 oa_config->sysfs_metric_id.attr.name = "id"; in create_dynamic_oa_sysfs_entry()
2286 oa_config->sysfs_metric_id.attr.mode = 0444; in create_dynamic_oa_sysfs_entry()
2287 oa_config->sysfs_metric_id.show = show_dynamic_id; in create_dynamic_oa_sysfs_entry()
2288 oa_config->sysfs_metric_id.store = NULL; in create_dynamic_oa_sysfs_entry()
2290 oa_config->attrs[0] = &oa_config->sysfs_metric_id.attr; in create_dynamic_oa_sysfs_entry()
2291 oa_config->attrs[1] = NULL; in create_dynamic_oa_sysfs_entry()
2293 oa_config->sysfs_metric.name = oa_config->uuid; in create_dynamic_oa_sysfs_entry()
2294 oa_config->sysfs_metric.attrs = oa_config->attrs; in create_dynamic_oa_sysfs_entry()
2296 return sysfs_create_group(oa->metrics_kobj, &oa_config->sysfs_metric); in create_dynamic_oa_sysfs_entry()
2300 * xe_oa_add_config_ioctl - Adds one OA config
2312 struct xe_oa *oa = &xe->oa; in xe_oa_add_config_ioctl()
2316 struct xe_oa_reg *regs; in xe_oa_add_config_ioctl() local
2319 if (!oa->xe) { in xe_oa_add_config_ioctl()
2320 drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); in xe_oa_add_config_ioctl()
2321 return -ENODEV; in xe_oa_add_config_ioctl()
2325 drm_dbg(&oa->xe->drm, "Insufficient privileges to add xe OA config\n"); in xe_oa_add_config_ioctl()
2326 return -EACCES; in xe_oa_add_config_ioctl()
2330 if (XE_IOCTL_DBG(oa->xe, err)) in xe_oa_add_config_ioctl()
2331 return -EFAULT; in xe_oa_add_config_ioctl()
2333 if (XE_IOCTL_DBG(oa->xe, arg->extensions) || in xe_oa_add_config_ioctl()
2334 XE_IOCTL_DBG(oa->xe, !arg->regs_ptr) || in xe_oa_add_config_ioctl()
2335 XE_IOCTL_DBG(oa->xe, !arg->n_regs)) in xe_oa_add_config_ioctl()
2336 return -EINVAL; in xe_oa_add_config_ioctl()
2340 return -ENOMEM; in xe_oa_add_config_ioctl()
2342 oa_config->oa = oa; in xe_oa_add_config_ioctl()
2343 kref_init(&oa_config->ref); in xe_oa_add_config_ioctl()
2345 if (!uuid_is_valid(arg->uuid)) { in xe_oa_add_config_ioctl()
2346 drm_dbg(&oa->xe->drm, "Invalid uuid format for OA config\n"); in xe_oa_add_config_ioctl()
2347 err = -EINVAL; in xe_oa_add_config_ioctl()
2351 /* Last character in oa_config->uuid will be 0 because oa_config is kzalloc */ in xe_oa_add_config_ioctl()
2352 memcpy(oa_config->uuid, arg->uuid, sizeof(arg->uuid)); in xe_oa_add_config_ioctl()
2354 oa_config->regs_len = arg->n_regs; in xe_oa_add_config_ioctl()
2355 regs = xe_oa_alloc_regs(oa, xe_oa_is_valid_config_reg_addr, in xe_oa_add_config_ioctl()
2356 u64_to_user_ptr(arg->regs_ptr), in xe_oa_add_config_ioctl()
2357 arg->n_regs); in xe_oa_add_config_ioctl()
2358 if (IS_ERR(regs)) { in xe_oa_add_config_ioctl()
2359 drm_dbg(&oa->xe->drm, "Failed to create OA config for mux_regs\n"); in xe_oa_add_config_ioctl()
2360 err = PTR_ERR(regs); in xe_oa_add_config_ioctl()
2363 oa_config->regs = regs; in xe_oa_add_config_ioctl()
2365 err = mutex_lock_interruptible(&oa->metrics_lock); in xe_oa_add_config_ioctl()
2370 idr_for_each_entry(&oa->metrics_idr, tmp, id) { in xe_oa_add_config_ioctl()
2371 if (!strcmp(tmp->uuid, oa_config->uuid)) { in xe_oa_add_config_ioctl()
2372 drm_dbg(&oa->xe->drm, "OA config already exists with this uuid\n"); in xe_oa_add_config_ioctl()
2373 err = -EADDRINUSE; in xe_oa_add_config_ioctl()
2380 drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n"); in xe_oa_add_config_ioctl()
2384 oa_config->id = idr_alloc(&oa->metrics_idr, oa_config, 1, 0, GFP_KERNEL); in xe_oa_add_config_ioctl()
2385 if (oa_config->id < 0) { in xe_oa_add_config_ioctl()
2386 drm_dbg(&oa->xe->drm, "Failed to create sysfs entry for OA config\n"); in xe_oa_add_config_ioctl()
2387 err = oa_config->id; in xe_oa_add_config_ioctl()
2391 mutex_unlock(&oa->metrics_lock); in xe_oa_add_config_ioctl()
2393 drm_dbg(&oa->xe->drm, "Added config %s id=%i\n", oa_config->uuid, oa_config->id); in xe_oa_add_config_ioctl()
2395 return oa_config->id; in xe_oa_add_config_ioctl()
2398 mutex_unlock(&oa->metrics_lock); in xe_oa_add_config_ioctl()
2401 drm_dbg(&oa->xe->drm, "Failed to add new OA config\n"); in xe_oa_add_config_ioctl()
2406 * xe_oa_remove_config_ioctl - Removes one OA config
2414 struct xe_oa *oa = &xe->oa; in xe_oa_remove_config_ioctl()
2419 if (!oa->xe) { in xe_oa_remove_config_ioctl()
2420 drm_dbg(&xe->drm, "xe oa interface not available for this system\n"); in xe_oa_remove_config_ioctl()
2421 return -ENODEV; in xe_oa_remove_config_ioctl()
2425 drm_dbg(&oa->xe->drm, "Insufficient privileges to remove xe OA config\n"); in xe_oa_remove_config_ioctl()
2426 return -EACCES; in xe_oa_remove_config_ioctl()
2430 if (XE_IOCTL_DBG(oa->xe, ret)) in xe_oa_remove_config_ioctl()
2433 ret = mutex_lock_interruptible(&oa->metrics_lock); in xe_oa_remove_config_ioctl()
2437 oa_config = idr_find(&oa->metrics_idr, arg); in xe_oa_remove_config_ioctl()
2439 drm_dbg(&oa->xe->drm, "Failed to remove unknown OA config\n"); in xe_oa_remove_config_ioctl()
2440 ret = -ENOENT; in xe_oa_remove_config_ioctl()
2444 WARN_ON(arg != oa_config->id); in xe_oa_remove_config_ioctl()
2446 sysfs_remove_group(oa->metrics_kobj, &oa_config->sysfs_metric); in xe_oa_remove_config_ioctl()
2447 idr_remove(&oa->metrics_idr, arg); in xe_oa_remove_config_ioctl()
2449 mutex_unlock(&oa->metrics_lock); in xe_oa_remove_config_ioctl()
2451 drm_dbg(&oa->xe->drm, "Removed config %s id=%i\n", oa_config->uuid, oa_config->id); in xe_oa_remove_config_ioctl()
2458 mutex_unlock(&oa->metrics_lock); in xe_oa_remove_config_ioctl()
2466 if (!oa->metrics_kobj) in xe_oa_unregister()
2469 kobject_put(oa->metrics_kobj); in xe_oa_unregister()
2470 oa->metrics_kobj = NULL; in xe_oa_unregister()
2474 * xe_oa_register - Xe OA registration
2481 struct xe_oa *oa = &xe->oa; in xe_oa_register()
2483 if (!oa->xe) in xe_oa_register()
2486 oa->metrics_kobj = kobject_create_and_add("metrics", in xe_oa_register()
2487 &xe->drm.primary->kdev->kobj); in xe_oa_register()
2488 if (!oa->metrics_kobj) in xe_oa_register()
2489 return -ENOMEM; in xe_oa_register()
2491 return devm_add_action_or_reset(xe->drm.dev, xe_oa_unregister, oa); in xe_oa_register()
2506 if (GRAPHICS_VERx100(gt_to_xe(hwe->gt)) < 1270) in __hwe_oam_unit()
2509 xe_gt_WARN_ON(hwe->gt, xe_gt_is_main_type(hwe->gt)); in __hwe_oam_unit()
2511 if (GRAPHICS_VER(gt_to_xe(hwe->gt)) < 20) in __hwe_oam_unit()
2520 else if (hwe->engine_id == XE_HW_ENGINE_GSCCS0) in __hwe_oam_unit()
2522 else if (!IS_DGFX(gt_to_xe(hwe->gt))) in __hwe_oam_unit()
2524 else if (hwe->class == XE_ENGINE_CLASS_VIDEO_DECODE) in __hwe_oam_unit()
2525 return (hwe->instance / 2 & 0x1) + 1; in __hwe_oam_unit()
2526 else if (hwe->class == XE_ENGINE_CLASS_VIDEO_ENHANCE) in __hwe_oam_unit()
2527 return (hwe->instance & 0x1) + 1; in __hwe_oam_unit()
2534 switch (hwe->class) { in __hwe_oa_unit()
2587 int i, num_units = gt->oa.num_oa_units; in __xe_oa_init_oa_units()
2590 struct xe_oa_unit *u = >->oa.oa_unit[i]; in __xe_oa_init_oa_units()
2593 u->regs = __oag_regs(); in __xe_oa_init_oa_units()
2594 u->type = DRM_XE_OA_UNIT_TYPE_OAG; in __xe_oa_init_oa_units()
2597 u->regs = __oam_regs(oam_base_addr[i]); in __xe_oa_init_oa_units()
2598 u->type = i == XE_OAM_UNIT_SAG && GRAPHICS_VER(gt_to_xe(gt)) >= 20 ? in __xe_oa_init_oa_units()
2602 u->gt = gt; in __xe_oa_init_oa_units()
2604 xe_mmio_write32(>->mmio, u->regs.oa_ctrl, 0); in __xe_oa_init_oa_units()
2607 xe_mmio_write32(>->mmio, u->regs.oa_debug, in __xe_oa_init_oa_units()
2611 u->oa_unit_id = gt_to_xe(gt)->oa.oa_unit_ids++; in __xe_oa_init_oa_units()
2622 u = drmm_kcalloc(>_to_xe(gt)->drm, num_oa_units, sizeof(*u), GFP_KERNEL); in xe_oa_init_gt()
2624 return -ENOMEM; in xe_oa_init_gt()
2629 hwe->oa_unit = NULL; in xe_oa_init_gt()
2632 hwe->oa_unit = &u[index]; in xe_oa_init_gt()
2636 gt->oa.num_oa_units = num_oa_units; in xe_oa_init_gt()
2637 gt->oa.oa_unit = u; in xe_oa_init_gt()
2641 drmm_mutex_init(>_to_xe(gt)->drm, >->oa.gt_lock); in xe_oa_init_gt()
2654 for (i = 0; i < gt->oa.num_oa_units; i++) { in xe_oa_print_gt_oa_units()
2655 u = >->oa.oa_unit[i]; in xe_oa_print_gt_oa_units()
2660 if (xe_oa_unit_id(hwe) == u->oa_unit_id) in xe_oa_print_gt_oa_units()
2661 n += scnprintf(buf + n, sizeof(buf) - n, "%s ", hwe->name); in xe_oa_print_gt_oa_units()
2663 xe_gt_dbg(gt, "oa_unit %d, type %d, Engines: %s\n", u->oa_unit_id, u->type, buf); in xe_oa_print_gt_oa_units()
2672 for_each_gt(gt, oa->xe, gt_id) in xe_oa_print_oa_units()
2686 for_each_gt(gt, oa->xe, i) { in xe_oa_init_oa_units()
2699 __set_bit(format, oa->format_mask); in oa_format_add()
2704 if (GRAPHICS_VER(oa->xe) >= 20) { in xe_oa_init_supported_formats()
2717 } else if (GRAPHICS_VERx100(oa->xe) >= 1270) { in xe_oa_init_supported_formats()
2725 } else if (GRAPHICS_VERx100(oa->xe) >= 1255) { in xe_oa_init_supported_formats()
2733 xe_assert(oa->xe, GRAPHICS_VER(oa->xe) >= 12); in xe_oa_init_supported_formats()
2751 struct xe_oa *oa = &xe->oa; in xe_oa_fini()
2753 if (!oa->xe) in xe_oa_fini()
2756 idr_for_each(&oa->metrics_idr, destroy_config, oa); in xe_oa_fini()
2757 idr_destroy(&oa->metrics_idr); in xe_oa_fini()
2759 oa->xe = NULL; in xe_oa_fini()
2763 * xe_oa_init - OA initialization during device probe
2770 struct xe_oa *oa = &xe->oa; in xe_oa_init()
2780 oa->xe = xe; in xe_oa_init()
2781 oa->oa_formats = oa_formats; in xe_oa_init()
2783 drmm_mutex_init(&oa->xe->drm, &oa->metrics_lock); in xe_oa_init()
2784 idr_init_base(&oa->metrics_idr, 1); in xe_oa_init()
2788 drm_err(&xe->drm, "OA initialization failed (%pe)\n", ERR_PTR(ret)); in xe_oa_init()
2794 return devm_add_action_or_reset(xe->drm.dev, xe_oa_fini, xe); in xe_oa_init()
2797 oa->xe = NULL; in xe_oa_init()