1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright 2023 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #ifndef AMDGPU_USERQ_H_ 26 #define AMDGPU_USERQ_H_ 27 #include "amdgpu_eviction_fence.h" 28 29 #define AMDGPU_MAX_USERQ_COUNT 512 30 31 #define to_ev_fence(f) container_of(f, struct amdgpu_eviction_fence, base) 32 #define uq_mgr_to_fpriv(u) container_of(u, struct amdgpu_fpriv, userq_mgr) 33 #define work_to_uq_mgr(w, name) container_of(w, struct amdgpu_userq_mgr, name) 34 35 enum amdgpu_userq_state { 36 AMDGPU_USERQ_STATE_UNMAPPED = 0, 37 AMDGPU_USERQ_STATE_MAPPED, 38 AMDGPU_USERQ_STATE_PREEMPTED, 39 AMDGPU_USERQ_STATE_HUNG, 40 AMDGPU_USERQ_STATE_INVALID_VA, 41 }; 42 43 struct amdgpu_mqd_prop; 44 45 struct amdgpu_userq_obj { 46 void *cpu_ptr; 47 uint64_t gpu_addr; 48 struct amdgpu_bo *obj; 49 }; 50 51 struct amdgpu_usermode_queue { 52 int queue_type; 53 enum amdgpu_userq_state state; 54 uint64_t doorbell_handle; 55 uint64_t doorbell_index; 56 u32 doorbell_offset; 57 uint64_t flags; 58 struct amdgpu_mqd_prop *userq_prop; 59 struct amdgpu_userq_mgr *userq_mgr; 60 struct amdgpu_vm *vm; 61 struct amdgpu_userq_obj mqd; 62 struct amdgpu_userq_obj db_obj; 63 struct amdgpu_userq_obj fw_obj; 64 struct amdgpu_userq_obj wptr_obj; 65 66 /** 67 * @fence_drv_lock: Protecting @fence_drv_xa. 68 */ 69 struct mutex fence_drv_lock; 70 71 /** 72 * @fence_drv_xa: 73 * 74 * References to the external fence drivers returned by wait_ioctl. 75 * Dropped on the next signaled dma_fence or queue destruction. 76 */ 77 struct xarray fence_drv_xa; 78 struct amdgpu_userq_fence_driver *fence_drv; 79 struct dma_fence *last_fence; 80 u32 xcp_id; 81 int priority; 82 struct dentry *debugfs_queue; 83 84 /** 85 * @hang_detect_work: 86 * 87 * Delayed work which runs when userq_fences time out. 88 */ 89 struct delayed_work hang_detect_work; 90 struct kref refcount; 91 92 union { 93 struct { 94 u64 queue_rb; 95 u64 wptr; 96 u64 rptr; 97 u64 eop; 98 u64 shadow; 99 u64 csa; 100 } va; 101 u64 va_array[6]; 102 } userq_vas; 103 104 uint32_t gang_ctx_array_index; 105 }; 106 107 struct amdgpu_userq_funcs { 108 int (*mqd_create)(struct amdgpu_usermode_queue *queue, 109 struct drm_amdgpu_userq_in *args); 110 int (*mqd_update)(struct amdgpu_usermode_queue *queue, 111 struct drm_amdgpu_userq_in *args); 112 void (*mqd_destroy)(struct amdgpu_usermode_queue *uq); 113 int (*unmap)(struct amdgpu_usermode_queue *queue); 114 int (*map)(struct amdgpu_usermode_queue *queue); 115 int (*preempt)(struct amdgpu_usermode_queue *queue); 116 int (*restore)(struct amdgpu_usermode_queue *queue); 117 int (*reset)(struct amdgpu_usermode_queue *queue); 118 }; 119 120 /* Usermode queues for gfx */ 121 struct amdgpu_userq_mgr { 122 /** 123 * @userq_xa: Per-process user queue map (queue ID → queue) 124 * Key: queue_id (unique ID within the process's userq manager) 125 * Value: struct amdgpu_usermode_queue 126 */ 127 struct xarray userq_xa; 128 struct mutex userq_mutex; 129 struct amdgpu_device *adev; 130 struct delayed_work resume_work; 131 struct drm_file *file; 132 struct mutex proc_ctx_lock; 133 struct amdgpu_userq_obj proc_ctx_obj; 134 135 bool proc_ctx_allocated; 136 uint32_t proc_ctx_array_index; 137 /** 138 * @reset_work: 139 * 140 * Reset work which is used when eviction fails. 141 */ 142 struct work_struct reset_work; 143 atomic_t userq_count[AMDGPU_RING_TYPE_MAX]; 144 }; 145 146 struct amdgpu_db_info { 147 uint64_t doorbell_handle; 148 uint32_t queue_type; 149 uint32_t doorbell_offset; 150 struct amdgpu_userq_obj *db_obj; 151 }; 152 153 struct amdgpu_usermode_queue *amdgpu_userq_get(struct amdgpu_userq_mgr *uq_mgr, u32 qid); 154 void amdgpu_userq_put(struct amdgpu_usermode_queue *queue); 155 156 int amdgpu_userq_ioctl(struct drm_device *dev, void *data, struct drm_file *filp); 157 158 int amdgpu_userq_mgr_init(struct amdgpu_userq_mgr *userq_mgr, struct drm_file *file_priv, 159 struct amdgpu_device *adev); 160 161 void amdgpu_userq_mgr_cancel_reset_work(struct amdgpu_device *adev); 162 void amdgpu_userq_mgr_cancel_resume(struct amdgpu_userq_mgr *userq_mgr); 163 void amdgpu_userq_mgr_fini(struct amdgpu_userq_mgr *userq_mgr); 164 165 void amdgpu_userq_evict(struct amdgpu_userq_mgr *uq_mgr); 166 167 void amdgpu_userq_ensure_ev_fence(struct amdgpu_userq_mgr *userq_mgr, 168 struct amdgpu_eviction_fence_mgr *evf_mgr); 169 170 u32 amdgpu_userq_get_supported_ip_mask(struct amdgpu_device *adev); 171 bool amdgpu_userq_enabled(struct drm_device *dev); 172 173 int amdgpu_userq_suspend(struct amdgpu_device *adev); 174 int amdgpu_userq_resume(struct amdgpu_device *adev); 175 176 int amdgpu_userq_stop_sched_for_enforce_isolation(struct amdgpu_device *adev, 177 u32 idx); 178 int amdgpu_userq_start_sched_for_enforce_isolation(struct amdgpu_device *adev, 179 u32 idx); 180 void amdgpu_userq_reset_work(struct work_struct *work); 181 void amdgpu_userq_pre_reset(struct amdgpu_device *adev); 182 int amdgpu_userq_post_reset(struct amdgpu_device *adev, bool vram_lost); 183 void amdgpu_userq_start_hang_detect_work(struct amdgpu_usermode_queue *queue); 184 void amdgpu_userq_process_fence_irq(struct amdgpu_device *adev, u32 doorbell); 185 186 /* 187 * CP packs the per-process doorbell_id of the queue in 188 * CTXID0[9:0] on priv-fault (same encoding KFD uses via 189 * KFD_CTXID0_DOORBELL_ID_MASK) 190 */ 191 #define AMDGPU_CTXID0_DOORBELL_ID_MASK 0x3ff 192 193 void amdgpu_userq_process_reset_irq(struct amdgpu_device *adev, 194 u32 pasid, u32 doorbell_offset); 195 196 int amdgpu_userq_input_va_validate(struct amdgpu_device *adev, 197 struct amdgpu_usermode_queue *queue, 198 u64 addr, u64 expected_size, u64 *va_out); 199 200 void amdgpu_userq_gem_va_unmap_validate(struct amdgpu_device *adev, 201 struct amdgpu_bo_va_mapping *mapping); 202 #endif 203