1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /* 3 * Copyright 2014-2022 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 25 #ifndef KFD_DEVICE_QUEUE_MANAGER_H_ 26 #define KFD_DEVICE_QUEUE_MANAGER_H_ 27 28 #include <linux/rwsem.h> 29 #include <linux/list.h> 30 #include <linux/mutex.h> 31 #include <linux/sched/mm.h> 32 #include "kfd_priv.h" 33 #include "kfd_mqd_manager.h" 34 35 #define VMID_NUM 16 36 37 #define KFD_MES_PROCESS_QUANTUM 100000 38 #define KFD_MES_GANG_QUANTUM 10000 39 40 struct device_process_node { 41 struct qcm_process_device *qpd; 42 struct list_head list; 43 }; 44 45 union SQ_CMD_BITS { 46 struct { 47 uint32_t cmd:3; 48 uint32_t:1; 49 uint32_t mode:3; 50 uint32_t check_vmid:1; 51 uint32_t trap_id:3; 52 uint32_t:5; 53 uint32_t wave_id:4; 54 uint32_t simd_id:2; 55 uint32_t:2; 56 uint32_t queue_id:3; 57 uint32_t:1; 58 uint32_t vm_id:4; 59 } bitfields, bits; 60 uint32_t u32All; 61 signed int i32All; 62 float f32All; 63 }; 64 65 union GRBM_GFX_INDEX_BITS { 66 struct { 67 uint32_t instance_index:8; 68 uint32_t sh_index:8; 69 uint32_t se_index:8; 70 uint32_t:5; 71 uint32_t sh_broadcast_writes:1; 72 uint32_t instance_broadcast_writes:1; 73 uint32_t se_broadcast_writes:1; 74 } bitfields, bits; 75 uint32_t u32All; 76 signed int i32All; 77 float f32All; 78 }; 79 80 /** 81 * struct device_queue_manager_ops 82 * 83 * @create_queue: Queue creation routine. 84 * 85 * @destroy_queue: Queue destruction routine. 86 * 87 * @update_queue: Queue update routine. 88 * 89 * @exeute_queues: Dispatches the queues list to the H/W. 90 * 91 * @register_process: This routine associates a specific process with device. 92 * 93 * @unregister_process: destroys the associations between process to device. 94 * 95 * @initialize: Initializes the pipelines and memory module for that device. 96 * 97 * @start: Initializes the resources/modules the device needs for queues 98 * execution. This function is called on device initialization and after the 99 * system woke up after suspension. 100 * 101 * @stop: This routine stops execution of all the active queue running on the 102 * H/W and basically this function called on system suspend. 103 * 104 * @uninitialize: Destroys all the device queue manager resources allocated in 105 * initialize routine. 106 * 107 * @halt: This routine unmaps queues from runlist and set halt status to true 108 * so no more queues will be mapped to runlist until unhalt. 109 * 110 * @unhalt: This routine unset halt status to flase and maps queues back to 111 * runlist. 112 * 113 * @create_kernel_queue: Creates kernel queue. Used for debug queue. 114 * 115 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue. 116 * 117 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the 118 * memory apertures. 119 * 120 * @process_termination: Clears all process queues belongs to that device. 121 * 122 * @evict_process_queues: Evict all active queues of a process 123 * 124 * @restore_process_queues: Restore all evicted queues of a process 125 * 126 * @get_wave_state: Retrieves context save state and optionally copies the 127 * control stack, if kept in the MQD, to the given userspace address. 128 * 129 * @reset_queues: reset queues which consume RAS poison 130 * @get_queue_checkpoint_info: Retrieves queue size information for CRIU checkpoint. 131 * 132 * @checkpoint_mqd: checkpoint queue MQD contents for CRIU. 133 */ 134 135 struct device_queue_manager_ops { 136 int (*create_queue)(struct device_queue_manager *dqm, 137 struct queue *q, 138 struct qcm_process_device *qpd, 139 const struct kfd_criu_queue_priv_data *qd, 140 const void *restore_mqd, 141 const void *restore_ctl_stack); 142 143 int (*destroy_queue)(struct device_queue_manager *dqm, 144 struct qcm_process_device *qpd, 145 struct queue *q); 146 147 int (*update_queue)(struct device_queue_manager *dqm, 148 struct queue *q, struct mqd_update_info *minfo); 149 150 int (*register_process)(struct device_queue_manager *dqm, 151 struct qcm_process_device *qpd); 152 153 int (*unregister_process)(struct device_queue_manager *dqm, 154 struct qcm_process_device *qpd); 155 156 int (*initialize)(struct device_queue_manager *dqm); 157 int (*start)(struct device_queue_manager *dqm); 158 int (*stop)(struct device_queue_manager *dqm); 159 void (*uninitialize)(struct device_queue_manager *dqm); 160 int (*halt)(struct device_queue_manager *dqm); 161 int (*unhalt)(struct device_queue_manager *dqm); 162 int (*create_kernel_queue)(struct device_queue_manager *dqm, 163 struct kernel_queue *kq, 164 struct qcm_process_device *qpd); 165 166 void (*destroy_kernel_queue)(struct device_queue_manager *dqm, 167 struct kernel_queue *kq, 168 struct qcm_process_device *qpd); 169 170 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, 171 struct qcm_process_device *qpd, 172 enum cache_policy default_policy, 173 enum cache_policy alternate_policy, 174 void __user *alternate_aperture_base, 175 uint64_t alternate_aperture_size, 176 u32 misc_process_properties); 177 178 int (*process_termination)(struct device_queue_manager *dqm, 179 struct qcm_process_device *qpd); 180 181 int (*evict_process_queues)(struct device_queue_manager *dqm, 182 struct qcm_process_device *qpd); 183 int (*restore_process_queues)(struct device_queue_manager *dqm, 184 struct qcm_process_device *qpd); 185 186 int (*get_wave_state)(struct device_queue_manager *dqm, 187 struct queue *q, 188 void __user *ctl_stack, 189 u32 *ctl_stack_used_size, 190 u32 *save_area_used_size); 191 192 int (*reset_queues)(struct device_queue_manager *dqm, 193 uint16_t pasid); 194 int (*get_queue_checkpoint_info)(struct device_queue_manager *dqm, 195 const struct queue *q, u32 *mqd_size, 196 u32 *ctl_stack_size); 197 198 int (*checkpoint_mqd)(struct device_queue_manager *dqm, 199 const struct queue *q, 200 void *mqd, 201 void *ctl_stack); 202 void (*set_perfcount)(struct device_queue_manager *dqm, 203 int enable); 204 }; 205 206 struct device_queue_manager_asic_ops { 207 int (*update_qpd)(struct device_queue_manager *dqm, 208 struct qcm_process_device *qpd); 209 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, 210 struct qcm_process_device *qpd, 211 enum cache_policy default_policy, 212 enum cache_policy alternate_policy, 213 void __user *alternate_aperture_base, 214 uint64_t alternate_aperture_size, 215 u32 misc_process_properties); 216 void (*init_sdma_vm)(struct device_queue_manager *dqm, 217 struct queue *q, 218 struct qcm_process_device *qpd); 219 struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type, 220 struct kfd_node *dev); 221 }; 222 223 struct dqm_detect_hang_info { 224 int pipe_id; 225 int queue_id; 226 int xcc_id; 227 uint64_t queue_address; 228 }; 229 230 /** 231 * struct device_queue_manager 232 * 233 * This struct is a base class for the kfd queues scheduler in the 234 * device level. The device base class should expose the basic operations 235 * for queue creation and queue destruction. This base class hides the 236 * scheduling mode of the driver and the specific implementation of the 237 * concrete device. This class is the only class in the queues scheduler 238 * that configures the H/W. 239 * 240 */ 241 242 struct device_queue_manager { 243 struct device_queue_manager_ops ops; 244 struct device_queue_manager_asic_ops asic_ops; 245 246 struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX]; 247 struct packet_manager packet_mgr; 248 struct kfd_node *dev; 249 struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */ 250 struct list_head queues; 251 unsigned int saved_flags; 252 unsigned int processes_count; 253 unsigned int active_queue_count; 254 unsigned int active_cp_queue_count; 255 unsigned int gws_queue_count; 256 unsigned int total_queue_count; 257 unsigned int next_pipe_to_allocate; 258 unsigned int *allocated_queues; 259 DECLARE_BITMAP(sdma_bitmap, KFD_MAX_SDMA_QUEUES); 260 DECLARE_BITMAP(xgmi_sdma_bitmap, KFD_MAX_SDMA_QUEUES); 261 /* the pasid mapping for each kfd vmid */ 262 uint16_t vmid_pasid[VMID_NUM]; 263 uint64_t pipelines_addr; 264 uint64_t fence_gpu_addr; 265 uint64_t *fence_addr; 266 struct kfd_mem_obj *fence_mem; 267 bool active_runlist; 268 int sched_policy; 269 uint32_t trap_debug_vmid; 270 271 /* hw exception */ 272 bool is_hws_hang; 273 bool is_resetting; 274 struct kfd_mem_obj hiq_sdma_mqd; 275 bool sched_running; 276 bool sched_halt; 277 278 /* used for GFX 9.4.3 only */ 279 uint32_t current_logical_xcc_start; 280 281 uint32_t wait_times; 282 283 wait_queue_head_t destroy_wait; 284 285 /* for per-queue reset support */ 286 struct dqm_detect_hang_info *detect_hang_info; 287 size_t detect_hang_info_size; 288 int detect_hang_count; 289 /* for per-queue reset with mes */ 290 u32 *hung_db_array; 291 struct amdgpu_mes_hung_queue_hqd_info *hqd_info; 292 }; 293 294 void device_queue_manager_init_cik( 295 struct device_queue_manager_asic_ops *asic_ops); 296 void device_queue_manager_init_vi( 297 struct device_queue_manager_asic_ops *asic_ops); 298 void device_queue_manager_init_v9( 299 struct device_queue_manager_asic_ops *asic_ops); 300 void device_queue_manager_init_v10( 301 struct device_queue_manager_asic_ops *asic_ops); 302 void device_queue_manager_init_v11( 303 struct device_queue_manager_asic_ops *asic_ops); 304 void device_queue_manager_init_v12( 305 struct device_queue_manager_asic_ops *asic_ops); 306 void device_queue_manager_init_v12_1( 307 struct device_queue_manager_asic_ops *asic_ops); 308 void program_sh_mem_settings(struct device_queue_manager *dqm, 309 struct qcm_process_device *qpd); 310 unsigned int get_cp_queues_num(struct device_queue_manager *dqm); 311 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm); 312 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm); 313 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm); 314 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm); 315 int reserve_debug_trap_vmid(struct device_queue_manager *dqm, 316 struct qcm_process_device *qpd); 317 int release_debug_trap_vmid(struct device_queue_manager *dqm, 318 struct qcm_process_device *qpd); 319 int suspend_queues(struct kfd_process *p, 320 uint32_t num_queues, 321 uint32_t grace_period, 322 uint64_t exception_clear_mask, 323 uint32_t *usr_queue_id_array); 324 int resume_queues(struct kfd_process *p, 325 uint32_t num_queues, 326 uint32_t *usr_queue_id_array); 327 void set_queue_snapshot_entry(struct queue *q, 328 uint64_t exception_clear_mask, 329 struct kfd_queue_snapshot_entry *qss_entry); 330 int debug_lock_and_unmap(struct device_queue_manager *dqm); 331 int debug_map_and_unlock(struct device_queue_manager *dqm); 332 int debug_refresh_runlist(struct device_queue_manager *dqm); 333 bool kfd_dqm_is_queue_in_process(struct device_queue_manager *dqm, 334 struct qcm_process_device *qpd, 335 int doorbell_off, u32 *queue_format); 336 337 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) 338 { 339 return (pdd->lds_base >> 16) & 0xFF; 340 } 341 342 static inline unsigned int 343 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd) 344 { 345 return (pdd->lds_base >> 60) & 0x0E; 346 } 347 348 /* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS 349 * happens while holding this lock anywhere to prevent deadlocks when 350 * an MMU notifier runs in reclaim-FS context. 351 */ 352 static inline void dqm_lock(struct device_queue_manager *dqm) 353 { 354 mutex_lock(&dqm->lock_hidden); 355 dqm->saved_flags = memalloc_noreclaim_save(); 356 } 357 static inline void dqm_unlock(struct device_queue_manager *dqm) 358 { 359 memalloc_noreclaim_restore(dqm->saved_flags); 360 mutex_unlock(&dqm->lock_hidden); 361 } 362 363 static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val) 364 { 365 /* SDMA activity counter is stored at queue's RPTR + 0x8 location. */ 366 return get_user(*val, q_rptr + 1); 367 } 368 369 static inline void update_dqm_wait_times(struct device_queue_manager *dqm) 370 { 371 if (dqm->dev->kfd2kgd->get_iq_wait_times) 372 dqm->dev->kfd2kgd->get_iq_wait_times(dqm->dev->adev, 373 &dqm->wait_times, 374 ffs(dqm->dev->xcc_mask) - 1); 375 } 376 377 378 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ 379