1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #ifndef KFD_DEVICE_QUEUE_MANAGER_H_ 25 #define KFD_DEVICE_QUEUE_MANAGER_H_ 26 27 #include <linux/rwsem.h> 28 #include <linux/list.h> 29 #include <linux/mutex.h> 30 #include <linux/sched/mm.h> 31 #include "kfd_priv.h" 32 #include "kfd_mqd_manager.h" 33 34 35 #define VMID_NUM 16 36 37 struct device_process_node { 38 struct qcm_process_device *qpd; 39 struct list_head list; 40 }; 41 42 /** 43 * struct device_queue_manager_ops 44 * 45 * @create_queue: Queue creation routine. 46 * 47 * @destroy_queue: Queue destruction routine. 48 * 49 * @update_queue: Queue update routine. 50 * 51 * @exeute_queues: Dispatches the queues list to the H/W. 52 * 53 * @register_process: This routine associates a specific process with device. 54 * 55 * @unregister_process: destroys the associations between process to device. 56 * 57 * @initialize: Initializes the pipelines and memory module for that device. 58 * 59 * @start: Initializes the resources/modules the the device needs for queues 60 * execution. This function is called on device initialization and after the 61 * system woke up after suspension. 62 * 63 * @stop: This routine stops execution of all the active queue running on the 64 * H/W and basically this function called on system suspend. 65 * 66 * @uninitialize: Destroys all the device queue manager resources allocated in 67 * initialize routine. 68 * 69 * @create_kernel_queue: Creates kernel queue. Used for debug queue. 70 * 71 * @destroy_kernel_queue: Destroys kernel queue. Used for debug queue. 72 * 73 * @set_cache_memory_policy: Sets memory policy (cached/ non cached) for the 74 * memory apertures. 75 * 76 * @process_termination: Clears all process queues belongs to that device. 77 * 78 * @evict_process_queues: Evict all active queues of a process 79 * 80 * @restore_process_queues: Restore all evicted queues queues of a process 81 * 82 * @get_wave_state: Retrieves context save state and optionally copies the 83 * control stack, if kept in the MQD, to the given userspace address. 84 */ 85 86 struct device_queue_manager_ops { 87 int (*create_queue)(struct device_queue_manager *dqm, 88 struct queue *q, 89 struct qcm_process_device *qpd); 90 91 int (*destroy_queue)(struct device_queue_manager *dqm, 92 struct qcm_process_device *qpd, 93 struct queue *q); 94 95 int (*update_queue)(struct device_queue_manager *dqm, 96 struct queue *q); 97 98 int (*register_process)(struct device_queue_manager *dqm, 99 struct qcm_process_device *qpd); 100 101 int (*unregister_process)(struct device_queue_manager *dqm, 102 struct qcm_process_device *qpd); 103 104 int (*initialize)(struct device_queue_manager *dqm); 105 int (*start)(struct device_queue_manager *dqm); 106 int (*stop)(struct device_queue_manager *dqm); 107 void (*pre_reset)(struct device_queue_manager *dqm); 108 void (*uninitialize)(struct device_queue_manager *dqm); 109 int (*create_kernel_queue)(struct device_queue_manager *dqm, 110 struct kernel_queue *kq, 111 struct qcm_process_device *qpd); 112 113 void (*destroy_kernel_queue)(struct device_queue_manager *dqm, 114 struct kernel_queue *kq, 115 struct qcm_process_device *qpd); 116 117 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, 118 struct qcm_process_device *qpd, 119 enum cache_policy default_policy, 120 enum cache_policy alternate_policy, 121 void __user *alternate_aperture_base, 122 uint64_t alternate_aperture_size); 123 124 int (*set_trap_handler)(struct device_queue_manager *dqm, 125 struct qcm_process_device *qpd, 126 uint64_t tba_addr, 127 uint64_t tma_addr); 128 129 int (*process_termination)(struct device_queue_manager *dqm, 130 struct qcm_process_device *qpd); 131 132 int (*evict_process_queues)(struct device_queue_manager *dqm, 133 struct qcm_process_device *qpd); 134 int (*restore_process_queues)(struct device_queue_manager *dqm, 135 struct qcm_process_device *qpd); 136 137 int (*get_wave_state)(struct device_queue_manager *dqm, 138 struct queue *q, 139 void __user *ctl_stack, 140 u32 *ctl_stack_used_size, 141 u32 *save_area_used_size); 142 }; 143 144 struct device_queue_manager_asic_ops { 145 int (*update_qpd)(struct device_queue_manager *dqm, 146 struct qcm_process_device *qpd); 147 bool (*set_cache_memory_policy)(struct device_queue_manager *dqm, 148 struct qcm_process_device *qpd, 149 enum cache_policy default_policy, 150 enum cache_policy alternate_policy, 151 void __user *alternate_aperture_base, 152 uint64_t alternate_aperture_size); 153 void (*init_sdma_vm)(struct device_queue_manager *dqm, 154 struct queue *q, 155 struct qcm_process_device *qpd); 156 struct mqd_manager * (*mqd_manager_init)(enum KFD_MQD_TYPE type, 157 struct kfd_dev *dev); 158 }; 159 160 /** 161 * struct device_queue_manager 162 * 163 * This struct is a base class for the kfd queues scheduler in the 164 * device level. The device base class should expose the basic operations 165 * for queue creation and queue destruction. This base class hides the 166 * scheduling mode of the driver and the specific implementation of the 167 * concrete device. This class is the only class in the queues scheduler 168 * that configures the H/W. 169 * 170 */ 171 172 struct device_queue_manager { 173 struct device_queue_manager_ops ops; 174 struct device_queue_manager_asic_ops asic_ops; 175 176 struct mqd_manager *mqd_mgrs[KFD_MQD_TYPE_MAX]; 177 struct packet_manager packets; 178 struct kfd_dev *dev; 179 struct mutex lock_hidden; /* use dqm_lock/unlock(dqm) */ 180 struct list_head queues; 181 unsigned int saved_flags; 182 unsigned int processes_count; 183 unsigned int active_queue_count; 184 unsigned int active_cp_queue_count; 185 unsigned int gws_queue_count; 186 unsigned int total_queue_count; 187 unsigned int next_pipe_to_allocate; 188 unsigned int *allocated_queues; 189 uint64_t sdma_bitmap; 190 uint64_t xgmi_sdma_bitmap; 191 /* the pasid mapping for each kfd vmid */ 192 uint16_t vmid_pasid[VMID_NUM]; 193 uint64_t pipelines_addr; 194 uint64_t fence_gpu_addr; 195 unsigned int *fence_addr; 196 struct kfd_mem_obj *fence_mem; 197 bool active_runlist; 198 int sched_policy; 199 200 /* hw exception */ 201 bool is_hws_hang; 202 bool is_resetting; 203 struct work_struct hw_exception_work; 204 struct kfd_mem_obj hiq_sdma_mqd; 205 bool sched_running; 206 }; 207 208 void device_queue_manager_init_cik( 209 struct device_queue_manager_asic_ops *asic_ops); 210 void device_queue_manager_init_cik_hawaii( 211 struct device_queue_manager_asic_ops *asic_ops); 212 void device_queue_manager_init_vi( 213 struct device_queue_manager_asic_ops *asic_ops); 214 void device_queue_manager_init_vi_tonga( 215 struct device_queue_manager_asic_ops *asic_ops); 216 void device_queue_manager_init_v9( 217 struct device_queue_manager_asic_ops *asic_ops); 218 void device_queue_manager_init_v10_navi10( 219 struct device_queue_manager_asic_ops *asic_ops); 220 void program_sh_mem_settings(struct device_queue_manager *dqm, 221 struct qcm_process_device *qpd); 222 unsigned int get_cp_queues_num(struct device_queue_manager *dqm); 223 unsigned int get_queues_per_pipe(struct device_queue_manager *dqm); 224 unsigned int get_pipes_per_mec(struct device_queue_manager *dqm); 225 unsigned int get_num_sdma_queues(struct device_queue_manager *dqm); 226 unsigned int get_num_xgmi_sdma_queues(struct device_queue_manager *dqm); 227 228 static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd) 229 { 230 return (pdd->lds_base >> 16) & 0xFF; 231 } 232 233 static inline unsigned int 234 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd) 235 { 236 return (pdd->lds_base >> 60) & 0x0E; 237 } 238 239 /* The DQM lock can be taken in MMU notifiers. Make sure no reclaim-FS 240 * happens while holding this lock anywhere to prevent deadlocks when 241 * an MMU notifier runs in reclaim-FS context. 242 */ 243 static inline void dqm_lock(struct device_queue_manager *dqm) 244 { 245 mutex_lock(&dqm->lock_hidden); 246 dqm->saved_flags = memalloc_nofs_save(); 247 } 248 static inline void dqm_unlock(struct device_queue_manager *dqm) 249 { 250 memalloc_nofs_restore(dqm->saved_flags); 251 mutex_unlock(&dqm->lock_hidden); 252 } 253 254 static inline int read_sdma_queue_counter(uint64_t __user *q_rptr, uint64_t *val) 255 { 256 /* 257 * SDMA activity counter is stored at queue's RPTR + 0x8 location. 258 */ 259 return get_user(*val, q_rptr + 1); 260 } 261 #endif /* KFD_DEVICE_QUEUE_MANAGER_H_ */ 262