1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 */ 23 24 #include <linux/printk.h> 25 #include <linux/slab.h> 26 #include "kfd_priv.h" 27 #include "kfd_mqd_manager.h" 28 #include "cik_regs.h" 29 #include "cik_structs.h" 30 31 static inline struct cik_mqd *get_mqd(void *mqd) 32 { 33 return (struct cik_mqd *)mqd; 34 } 35 36 static int init_mqd(struct mqd_manager *mm, void **mqd, 37 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 38 struct queue_properties *q) 39 { 40 uint64_t addr; 41 struct cik_mqd *m; 42 int retval; 43 44 BUG_ON(!mm || !q || !mqd); 45 46 pr_debug("kfd: In func %s\n", __func__); 47 48 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), 49 mqd_mem_obj); 50 51 if (retval != 0) 52 return -ENOMEM; 53 54 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; 55 addr = (*mqd_mem_obj)->gpu_addr; 56 57 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); 58 59 m->header = 0xC0310800; 60 m->compute_pipelinestat_enable = 1; 61 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; 62 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; 63 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; 64 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; 65 66 /* 67 * Make sure to use the last queue state saved on mqd when the cp 68 * reassigns the queue, so when queue is switched on/off (e.g over 69 * subscription or quantum timeout) the context will be consistent 70 */ 71 m->cp_hqd_persistent_state = 72 DEFAULT_CP_HQD_PERSISTENT_STATE | PRELOAD_REQ; 73 74 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; 75 m->cp_mqd_base_addr_lo = lower_32_bits(addr); 76 m->cp_mqd_base_addr_hi = upper_32_bits(addr); 77 78 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE | IB_ATC_EN; 79 /* Although WinKFD writes this, I suspect it should not be necessary */ 80 m->cp_hqd_ib_control = IB_ATC_EN | DEFAULT_MIN_IB_AVAIL_SIZE; 81 82 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | 83 QUANTUM_DURATION(10); 84 85 /* 86 * Pipe Priority 87 * Identifies the pipe relative priority when this queue is connected 88 * to the pipeline. The pipe priority is against the GFX pipe and HP3D. 89 * In KFD we are using a fixed pipe priority set to CS_MEDIUM. 90 * 0 = CS_LOW (typically below GFX) 91 * 1 = CS_MEDIUM (typically between HP3D and GFX 92 * 2 = CS_HIGH (typically above HP3D) 93 */ 94 m->cp_hqd_pipe_priority = 1; 95 m->cp_hqd_queue_priority = 15; 96 97 if (q->format == KFD_QUEUE_FORMAT_AQL) 98 m->cp_hqd_iq_rptr = AQL_ENABLE; 99 100 *mqd = m; 101 if (gart_addr != NULL) 102 *gart_addr = addr; 103 retval = mm->update_mqd(mm, m, q); 104 105 return retval; 106 } 107 108 static int init_mqd_sdma(struct mqd_manager *mm, void **mqd, 109 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 110 struct queue_properties *q) 111 { 112 int retval; 113 struct cik_sdma_rlc_registers *m; 114 115 BUG_ON(!mm || !mqd || !mqd_mem_obj); 116 117 retval = kfd_gtt_sa_allocate(mm->dev, 118 sizeof(struct cik_sdma_rlc_registers), 119 mqd_mem_obj); 120 121 if (retval != 0) 122 return -ENOMEM; 123 124 m = (struct cik_sdma_rlc_registers *) (*mqd_mem_obj)->cpu_ptr; 125 126 memset(m, 0, sizeof(struct cik_sdma_rlc_registers)); 127 128 *mqd = m; 129 if (gart_addr != NULL) 130 *gart_addr = (*mqd_mem_obj)->gpu_addr; 131 132 retval = mm->update_mqd(mm, m, q); 133 134 return retval; 135 } 136 137 static void uninit_mqd(struct mqd_manager *mm, void *mqd, 138 struct kfd_mem_obj *mqd_mem_obj) 139 { 140 BUG_ON(!mm || !mqd); 141 kfd_gtt_sa_free(mm->dev, mqd_mem_obj); 142 } 143 144 static void uninit_mqd_sdma(struct mqd_manager *mm, void *mqd, 145 struct kfd_mem_obj *mqd_mem_obj) 146 { 147 BUG_ON(!mm || !mqd); 148 kfd_gtt_sa_free(mm->dev, mqd_mem_obj); 149 } 150 151 static int load_mqd(struct mqd_manager *mm, void *mqd, uint32_t pipe_id, 152 uint32_t queue_id, uint32_t __user *wptr) 153 { 154 return kfd2kgd->hqd_load(mm->dev->kgd, mqd, pipe_id, queue_id, wptr); 155 } 156 157 static int load_mqd_sdma(struct mqd_manager *mm, void *mqd, 158 uint32_t pipe_id, uint32_t queue_id, 159 uint32_t __user *wptr) 160 { 161 return kfd2kgd->hqd_sdma_load(mm->dev->kgd, mqd); 162 } 163 164 static int update_mqd(struct mqd_manager *mm, void *mqd, 165 struct queue_properties *q) 166 { 167 struct cik_mqd *m; 168 169 BUG_ON(!mm || !q || !mqd); 170 171 pr_debug("kfd: In func %s\n", __func__); 172 173 m = get_mqd(mqd); 174 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | 175 DEFAULT_MIN_AVAIL_SIZE | PQ_ATC_EN; 176 177 /* 178 * Calculating queue size which is log base 2 of actual queue size -1 179 * dwords and another -1 for ffs 180 */ 181 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) 182 - 1 - 1; 183 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); 184 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); 185 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 186 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 187 m->cp_hqd_pq_doorbell_control = DOORBELL_EN | 188 DOORBELL_OFFSET(q->doorbell_off); 189 190 m->cp_hqd_vmid = q->vmid; 191 192 if (q->format == KFD_QUEUE_FORMAT_AQL) { 193 m->cp_hqd_pq_control |= NO_UPDATE_RPTR; 194 } 195 196 m->cp_hqd_active = 0; 197 q->is_active = false; 198 if (q->queue_size > 0 && 199 q->queue_address != 0 && 200 q->queue_percent > 0) { 201 m->cp_hqd_active = 1; 202 q->is_active = true; 203 } 204 205 return 0; 206 } 207 208 static int update_mqd_sdma(struct mqd_manager *mm, void *mqd, 209 struct queue_properties *q) 210 { 211 struct cik_sdma_rlc_registers *m; 212 213 BUG_ON(!mm || !mqd || !q); 214 215 m = get_sdma_mqd(mqd); 216 m->sdma_rlc_rb_cntl = 217 SDMA_RB_SIZE((ffs(q->queue_size / sizeof(unsigned int)))) | 218 SDMA_RB_VMID(q->vmid) | 219 SDMA_RPTR_WRITEBACK_ENABLE | 220 SDMA_RPTR_WRITEBACK_TIMER(6); 221 222 m->sdma_rlc_rb_base = lower_32_bits(q->queue_address >> 8); 223 m->sdma_rlc_rb_base_hi = upper_32_bits(q->queue_address >> 8); 224 m->sdma_rlc_rb_rptr_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 225 m->sdma_rlc_rb_rptr_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 226 m->sdma_rlc_doorbell = SDMA_OFFSET(q->doorbell_off) | SDMA_DB_ENABLE; 227 m->sdma_rlc_virtual_addr = q->sdma_vm_addr; 228 229 m->sdma_engine_id = q->sdma_engine_id; 230 m->sdma_queue_id = q->sdma_queue_id; 231 232 q->is_active = false; 233 if (q->queue_size > 0 && 234 q->queue_address != 0 && 235 q->queue_percent > 0) { 236 m->sdma_rlc_rb_cntl |= SDMA_RB_ENABLE; 237 q->is_active = true; 238 } 239 240 return 0; 241 } 242 243 static int destroy_mqd(struct mqd_manager *mm, void *mqd, 244 enum kfd_preempt_type type, 245 unsigned int timeout, uint32_t pipe_id, 246 uint32_t queue_id) 247 { 248 return kfd2kgd->hqd_destroy(mm->dev->kgd, type, timeout, 249 pipe_id, queue_id); 250 } 251 252 /* 253 * preempt type here is ignored because there is only one way 254 * to preempt sdma queue 255 */ 256 static int destroy_mqd_sdma(struct mqd_manager *mm, void *mqd, 257 enum kfd_preempt_type type, 258 unsigned int timeout, uint32_t pipe_id, 259 uint32_t queue_id) 260 { 261 return kfd2kgd->hqd_sdma_destroy(mm->dev->kgd, mqd, timeout); 262 } 263 264 static bool is_occupied(struct mqd_manager *mm, void *mqd, 265 uint64_t queue_address, uint32_t pipe_id, 266 uint32_t queue_id) 267 { 268 269 return kfd2kgd->hqd_is_occupied(mm->dev->kgd, queue_address, 270 pipe_id, queue_id); 271 272 } 273 274 static bool is_occupied_sdma(struct mqd_manager *mm, void *mqd, 275 uint64_t queue_address, uint32_t pipe_id, 276 uint32_t queue_id) 277 { 278 return kfd2kgd->hqd_sdma_is_occupied(mm->dev->kgd, mqd); 279 } 280 281 /* 282 * HIQ MQD Implementation, concrete implementation for HIQ MQD implementation. 283 * The HIQ queue in Kaveri is using the same MQD structure as all the user mode 284 * queues but with different initial values. 285 */ 286 287 static int init_mqd_hiq(struct mqd_manager *mm, void **mqd, 288 struct kfd_mem_obj **mqd_mem_obj, uint64_t *gart_addr, 289 struct queue_properties *q) 290 { 291 uint64_t addr; 292 struct cik_mqd *m; 293 int retval; 294 295 BUG_ON(!mm || !q || !mqd || !mqd_mem_obj); 296 297 pr_debug("kfd: In func %s\n", __func__); 298 299 retval = kfd_gtt_sa_allocate(mm->dev, sizeof(struct cik_mqd), 300 mqd_mem_obj); 301 302 if (retval != 0) 303 return -ENOMEM; 304 305 m = (struct cik_mqd *) (*mqd_mem_obj)->cpu_ptr; 306 addr = (*mqd_mem_obj)->gpu_addr; 307 308 memset(m, 0, ALIGN(sizeof(struct cik_mqd), 256)); 309 310 m->header = 0xC0310800; 311 m->compute_pipelinestat_enable = 1; 312 m->compute_static_thread_mgmt_se0 = 0xFFFFFFFF; 313 m->compute_static_thread_mgmt_se1 = 0xFFFFFFFF; 314 m->compute_static_thread_mgmt_se2 = 0xFFFFFFFF; 315 m->compute_static_thread_mgmt_se3 = 0xFFFFFFFF; 316 317 m->cp_hqd_persistent_state = DEFAULT_CP_HQD_PERSISTENT_STATE | 318 PRELOAD_REQ; 319 m->cp_hqd_quantum = QUANTUM_EN | QUANTUM_SCALE_1MS | 320 QUANTUM_DURATION(10); 321 322 m->cp_mqd_control = MQD_CONTROL_PRIV_STATE_EN; 323 m->cp_mqd_base_addr_lo = lower_32_bits(addr); 324 m->cp_mqd_base_addr_hi = upper_32_bits(addr); 325 326 m->cp_hqd_ib_control = DEFAULT_MIN_IB_AVAIL_SIZE; 327 328 /* 329 * Pipe Priority 330 * Identifies the pipe relative priority when this queue is connected 331 * to the pipeline. The pipe priority is against the GFX pipe and HP3D. 332 * In KFD we are using a fixed pipe priority set to CS_MEDIUM. 333 * 0 = CS_LOW (typically below GFX) 334 * 1 = CS_MEDIUM (typically between HP3D and GFX 335 * 2 = CS_HIGH (typically above HP3D) 336 */ 337 m->cp_hqd_pipe_priority = 1; 338 m->cp_hqd_queue_priority = 15; 339 340 *mqd = m; 341 if (gart_addr) 342 *gart_addr = addr; 343 retval = mm->update_mqd(mm, m, q); 344 345 return retval; 346 } 347 348 static int update_mqd_hiq(struct mqd_manager *mm, void *mqd, 349 struct queue_properties *q) 350 { 351 struct cik_mqd *m; 352 353 BUG_ON(!mm || !q || !mqd); 354 355 pr_debug("kfd: In func %s\n", __func__); 356 357 m = get_mqd(mqd); 358 m->cp_hqd_pq_control = DEFAULT_RPTR_BLOCK_SIZE | 359 DEFAULT_MIN_AVAIL_SIZE | 360 PRIV_STATE | 361 KMD_QUEUE; 362 363 /* 364 * Calculating queue size which is log base 2 of actual queue 365 * size -1 dwords 366 */ 367 m->cp_hqd_pq_control |= ffs(q->queue_size / sizeof(unsigned int)) 368 - 1 - 1; 369 m->cp_hqd_pq_base_lo = lower_32_bits((uint64_t)q->queue_address >> 8); 370 m->cp_hqd_pq_base_hi = upper_32_bits((uint64_t)q->queue_address >> 8); 371 m->cp_hqd_pq_rptr_report_addr_lo = lower_32_bits((uint64_t)q->read_ptr); 372 m->cp_hqd_pq_rptr_report_addr_hi = upper_32_bits((uint64_t)q->read_ptr); 373 m->cp_hqd_pq_doorbell_control = DOORBELL_EN | 374 DOORBELL_OFFSET(q->doorbell_off); 375 376 m->cp_hqd_vmid = q->vmid; 377 378 m->cp_hqd_active = 0; 379 q->is_active = false; 380 if (q->queue_size > 0 && 381 q->queue_address != 0 && 382 q->queue_percent > 0) { 383 m->cp_hqd_active = 1; 384 q->is_active = true; 385 } 386 387 return 0; 388 } 389 390 struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd) 391 { 392 struct cik_sdma_rlc_registers *m; 393 394 BUG_ON(!mqd); 395 396 m = (struct cik_sdma_rlc_registers *)mqd; 397 398 return m; 399 } 400 401 struct mqd_manager *mqd_manager_init_cik(enum KFD_MQD_TYPE type, 402 struct kfd_dev *dev) 403 { 404 struct mqd_manager *mqd; 405 406 BUG_ON(!dev); 407 BUG_ON(type >= KFD_MQD_TYPE_MAX); 408 409 pr_debug("kfd: In func %s\n", __func__); 410 411 mqd = kzalloc(sizeof(struct mqd_manager), GFP_KERNEL); 412 if (!mqd) 413 return NULL; 414 415 mqd->dev = dev; 416 417 switch (type) { 418 case KFD_MQD_TYPE_CP: 419 case KFD_MQD_TYPE_COMPUTE: 420 mqd->init_mqd = init_mqd; 421 mqd->uninit_mqd = uninit_mqd; 422 mqd->load_mqd = load_mqd; 423 mqd->update_mqd = update_mqd; 424 mqd->destroy_mqd = destroy_mqd; 425 mqd->is_occupied = is_occupied; 426 break; 427 case KFD_MQD_TYPE_HIQ: 428 mqd->init_mqd = init_mqd_hiq; 429 mqd->uninit_mqd = uninit_mqd; 430 mqd->load_mqd = load_mqd; 431 mqd->update_mqd = update_mqd_hiq; 432 mqd->destroy_mqd = destroy_mqd; 433 mqd->is_occupied = is_occupied; 434 break; 435 case KFD_MQD_TYPE_SDMA: 436 mqd->init_mqd = init_mqd_sdma; 437 mqd->uninit_mqd = uninit_mqd_sdma; 438 mqd->load_mqd = load_mqd_sdma; 439 mqd->update_mqd = update_mqd_sdma; 440 mqd->destroy_mqd = destroy_mqd_sdma; 441 mqd->is_occupied = is_occupied_sdma; 442 break; 443 default: 444 kfree(mqd); 445 return NULL; 446 } 447 448 return mqd; 449 } 450 451