1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "xe_execlist.h" 7 8 #include <drm/drm_managed.h> 9 10 #include "instructions/xe_mi_commands.h" 11 #include "regs/xe_engine_regs.h" 12 #include "regs/xe_gt_regs.h" 13 #include "regs/xe_lrc_layout.h" 14 #include "xe_assert.h" 15 #include "xe_bo.h" 16 #include "xe_device.h" 17 #include "xe_exec_queue.h" 18 #include "xe_gt.h" 19 #include "xe_hw_fence.h" 20 #include "xe_lrc.h" 21 #include "xe_macros.h" 22 #include "xe_mmio.h" 23 #include "xe_mocs.h" 24 #include "xe_ring_ops_types.h" 25 #include "xe_sched_job.h" 26 27 #define XE_EXECLIST_HANG_LIMIT 1 28 29 #define SW_CTX_ID_SHIFT 37 30 #define SW_CTX_ID_WIDTH 11 31 #define XEHP_SW_CTX_ID_SHIFT 39 32 #define XEHP_SW_CTX_ID_WIDTH 16 33 34 #define SW_CTX_ID \ 35 GENMASK_ULL(SW_CTX_ID_WIDTH + SW_CTX_ID_SHIFT - 1, \ 36 SW_CTX_ID_SHIFT) 37 38 #define XEHP_SW_CTX_ID \ 39 GENMASK_ULL(XEHP_SW_CTX_ID_WIDTH + XEHP_SW_CTX_ID_SHIFT - 1, \ 40 XEHP_SW_CTX_ID_SHIFT) 41 42 43 static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc, 44 u32 ctx_id) 45 { 46 struct xe_gt *gt = hwe->gt; 47 struct xe_device *xe = gt_to_xe(gt); 48 u64 lrc_desc; 49 50 lrc_desc = xe_lrc_descriptor(lrc); 51 52 if (GRAPHICS_VERx100(xe) >= 1250) { 53 xe_gt_assert(hwe->gt, FIELD_FIT(XEHP_SW_CTX_ID, ctx_id)); 54 lrc_desc |= FIELD_PREP(XEHP_SW_CTX_ID, ctx_id); 55 } else { 56 xe_gt_assert(hwe->gt, FIELD_FIT(SW_CTX_ID, ctx_id)); 57 lrc_desc |= FIELD_PREP(SW_CTX_ID, ctx_id); 58 } 59 60 if (hwe->class == XE_ENGINE_CLASS_COMPUTE) 61 xe_mmio_write32(hwe->gt, RCU_MODE, 62 _MASKED_BIT_ENABLE(RCU_MODE_CCS_ENABLE)); 63 64 xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); 65 lrc->ring.old_tail = lrc->ring.tail; 66 67 /* 68 * Make sure the context image is complete before we submit it to HW. 69 * 70 * Ostensibly, writes (including the WCB) should be flushed prior to 71 * an uncached write such as our mmio register access, the empirical 72 * evidence (esp. on Braswell) suggests that the WC write into memory 73 * may not be visible to the HW prior to the completion of the UC 74 * register write and that we may begin execution from the context 75 * before its image is complete leading to invalid PD chasing. 76 */ 77 wmb(); 78 79 xe_mmio_write32(gt, RING_HWS_PGA(hwe->mmio_base), 80 xe_bo_ggtt_addr(hwe->hwsp)); 81 xe_mmio_read32(gt, RING_HWS_PGA(hwe->mmio_base)); 82 xe_mmio_write32(gt, RING_MODE(hwe->mmio_base), 83 _MASKED_BIT_ENABLE(GFX_DISABLE_LEGACY_MODE)); 84 85 xe_mmio_write32(gt, RING_EXECLIST_SQ_CONTENTS_LO(hwe->mmio_base), 86 lower_32_bits(lrc_desc)); 87 xe_mmio_write32(gt, RING_EXECLIST_SQ_CONTENTS_HI(hwe->mmio_base), 88 upper_32_bits(lrc_desc)); 89 xe_mmio_write32(gt, RING_EXECLIST_CONTROL(hwe->mmio_base), 90 EL_CTRL_LOAD); 91 } 92 93 static void __xe_execlist_port_start(struct xe_execlist_port *port, 94 struct xe_execlist_exec_queue *exl) 95 { 96 struct xe_device *xe = gt_to_xe(port->hwe->gt); 97 int max_ctx = FIELD_MAX(SW_CTX_ID); 98 99 if (GRAPHICS_VERx100(xe) >= 1250) 100 max_ctx = FIELD_MAX(XEHP_SW_CTX_ID); 101 102 xe_execlist_port_assert_held(port); 103 104 if (port->running_exl != exl || !exl->has_run) { 105 port->last_ctx_id++; 106 107 /* 0 is reserved for the kernel context */ 108 if (port->last_ctx_id > max_ctx) 109 port->last_ctx_id = 1; 110 } 111 112 __start_lrc(port->hwe, exl->q->lrc[0], port->last_ctx_id); 113 port->running_exl = exl; 114 exl->has_run = true; 115 } 116 117 static void __xe_execlist_port_idle(struct xe_execlist_port *port) 118 { 119 u32 noop[2] = { MI_NOOP, MI_NOOP }; 120 121 xe_execlist_port_assert_held(port); 122 123 if (!port->running_exl) 124 return; 125 126 xe_lrc_write_ring(port->hwe->kernel_lrc, noop, sizeof(noop)); 127 __start_lrc(port->hwe, port->hwe->kernel_lrc, 0); 128 port->running_exl = NULL; 129 } 130 131 static bool xe_execlist_is_idle(struct xe_execlist_exec_queue *exl) 132 { 133 struct xe_lrc *lrc = exl->q->lrc[0]; 134 135 return lrc->ring.tail == lrc->ring.old_tail; 136 } 137 138 static void __xe_execlist_port_start_next_active(struct xe_execlist_port *port) 139 { 140 struct xe_execlist_exec_queue *exl = NULL; 141 int i; 142 143 xe_execlist_port_assert_held(port); 144 145 for (i = ARRAY_SIZE(port->active) - 1; i >= 0; i--) { 146 while (!list_empty(&port->active[i])) { 147 exl = list_first_entry(&port->active[i], 148 struct xe_execlist_exec_queue, 149 active_link); 150 list_del(&exl->active_link); 151 152 if (xe_execlist_is_idle(exl)) { 153 exl->active_priority = XE_EXEC_QUEUE_PRIORITY_UNSET; 154 continue; 155 } 156 157 list_add_tail(&exl->active_link, &port->active[i]); 158 __xe_execlist_port_start(port, exl); 159 return; 160 } 161 } 162 163 __xe_execlist_port_idle(port); 164 } 165 166 static u64 read_execlist_status(struct xe_hw_engine *hwe) 167 { 168 struct xe_gt *gt = hwe->gt; 169 u32 hi, lo; 170 171 lo = xe_mmio_read32(gt, RING_EXECLIST_STATUS_LO(hwe->mmio_base)); 172 hi = xe_mmio_read32(gt, RING_EXECLIST_STATUS_HI(hwe->mmio_base)); 173 174 return lo | (u64)hi << 32; 175 } 176 177 static void xe_execlist_port_irq_handler_locked(struct xe_execlist_port *port) 178 { 179 u64 status; 180 181 xe_execlist_port_assert_held(port); 182 183 status = read_execlist_status(port->hwe); 184 if (status & BIT(7)) 185 return; 186 187 __xe_execlist_port_start_next_active(port); 188 } 189 190 static void xe_execlist_port_irq_handler(struct xe_hw_engine *hwe, 191 u16 intr_vec) 192 { 193 struct xe_execlist_port *port = hwe->exl_port; 194 195 spin_lock(&port->lock); 196 xe_execlist_port_irq_handler_locked(port); 197 spin_unlock(&port->lock); 198 } 199 200 static void xe_execlist_port_wake_locked(struct xe_execlist_port *port, 201 enum xe_exec_queue_priority priority) 202 { 203 xe_execlist_port_assert_held(port); 204 205 if (port->running_exl && port->running_exl->active_priority >= priority) 206 return; 207 208 __xe_execlist_port_start_next_active(port); 209 } 210 211 static void xe_execlist_make_active(struct xe_execlist_exec_queue *exl) 212 { 213 struct xe_execlist_port *port = exl->port; 214 enum xe_exec_queue_priority priority = exl->q->sched_props.priority; 215 216 XE_WARN_ON(priority == XE_EXEC_QUEUE_PRIORITY_UNSET); 217 XE_WARN_ON(priority < 0); 218 XE_WARN_ON(priority >= ARRAY_SIZE(exl->port->active)); 219 220 spin_lock_irq(&port->lock); 221 222 if (exl->active_priority != priority && 223 exl->active_priority != XE_EXEC_QUEUE_PRIORITY_UNSET) { 224 /* Priority changed, move it to the right list */ 225 list_del(&exl->active_link); 226 exl->active_priority = XE_EXEC_QUEUE_PRIORITY_UNSET; 227 } 228 229 if (exl->active_priority == XE_EXEC_QUEUE_PRIORITY_UNSET) { 230 exl->active_priority = priority; 231 list_add_tail(&exl->active_link, &port->active[priority]); 232 } 233 234 xe_execlist_port_wake_locked(exl->port, priority); 235 236 spin_unlock_irq(&port->lock); 237 } 238 239 static void xe_execlist_port_irq_fail_timer(struct timer_list *timer) 240 { 241 struct xe_execlist_port *port = 242 container_of(timer, struct xe_execlist_port, irq_fail); 243 244 spin_lock_irq(&port->lock); 245 xe_execlist_port_irq_handler_locked(port); 246 spin_unlock_irq(&port->lock); 247 248 port->irq_fail.expires = jiffies + msecs_to_jiffies(1000); 249 add_timer(&port->irq_fail); 250 } 251 252 struct xe_execlist_port *xe_execlist_port_create(struct xe_device *xe, 253 struct xe_hw_engine *hwe) 254 { 255 struct drm_device *drm = &xe->drm; 256 struct xe_execlist_port *port; 257 int i; 258 259 port = drmm_kzalloc(drm, sizeof(*port), GFP_KERNEL); 260 if (!port) 261 return ERR_PTR(-ENOMEM); 262 263 port->hwe = hwe; 264 265 spin_lock_init(&port->lock); 266 for (i = 0; i < ARRAY_SIZE(port->active); i++) 267 INIT_LIST_HEAD(&port->active[i]); 268 269 port->last_ctx_id = 1; 270 port->running_exl = NULL; 271 272 hwe->irq_handler = xe_execlist_port_irq_handler; 273 274 /* TODO: Fix the interrupt code so it doesn't race like mad */ 275 timer_setup(&port->irq_fail, xe_execlist_port_irq_fail_timer, 0); 276 port->irq_fail.expires = jiffies + msecs_to_jiffies(1000); 277 add_timer(&port->irq_fail); 278 279 return port; 280 } 281 282 void xe_execlist_port_destroy(struct xe_execlist_port *port) 283 { 284 del_timer(&port->irq_fail); 285 286 /* Prevent an interrupt while we're destroying */ 287 spin_lock_irq(>_to_xe(port->hwe->gt)->irq.lock); 288 port->hwe->irq_handler = NULL; 289 spin_unlock_irq(>_to_xe(port->hwe->gt)->irq.lock); 290 } 291 292 static struct dma_fence * 293 execlist_run_job(struct drm_sched_job *drm_job) 294 { 295 struct xe_sched_job *job = to_xe_sched_job(drm_job); 296 struct xe_exec_queue *q = job->q; 297 struct xe_execlist_exec_queue *exl = job->q->execlist; 298 299 q->ring_ops->emit_job(job); 300 xe_execlist_make_active(exl); 301 302 return dma_fence_get(job->fence); 303 } 304 305 static void execlist_job_free(struct drm_sched_job *drm_job) 306 { 307 struct xe_sched_job *job = to_xe_sched_job(drm_job); 308 309 xe_exec_queue_update_run_ticks(job->q); 310 xe_sched_job_put(job); 311 } 312 313 static const struct drm_sched_backend_ops drm_sched_ops = { 314 .run_job = execlist_run_job, 315 .free_job = execlist_job_free, 316 }; 317 318 static int execlist_exec_queue_init(struct xe_exec_queue *q) 319 { 320 struct drm_gpu_scheduler *sched; 321 struct xe_execlist_exec_queue *exl; 322 struct xe_device *xe = gt_to_xe(q->gt); 323 int err; 324 325 xe_assert(xe, !xe_device_uc_enabled(xe)); 326 327 drm_info(&xe->drm, "Enabling execlist submission (GuC submission disabled)\n"); 328 329 exl = kzalloc(sizeof(*exl), GFP_KERNEL); 330 if (!exl) 331 return -ENOMEM; 332 333 exl->q = q; 334 335 err = drm_sched_init(&exl->sched, &drm_sched_ops, NULL, 1, 336 q->lrc[0]->ring.size / MAX_JOB_SIZE_BYTES, 337 XE_SCHED_HANG_LIMIT, XE_SCHED_JOB_TIMEOUT, 338 NULL, NULL, q->hwe->name, 339 gt_to_xe(q->gt)->drm.dev); 340 if (err) 341 goto err_free; 342 343 sched = &exl->sched; 344 err = drm_sched_entity_init(&exl->entity, 0, &sched, 1, NULL); 345 if (err) 346 goto err_sched; 347 348 exl->port = q->hwe->exl_port; 349 exl->has_run = false; 350 exl->active_priority = XE_EXEC_QUEUE_PRIORITY_UNSET; 351 q->execlist = exl; 352 q->entity = &exl->entity; 353 354 xe_exec_queue_assign_name(q, ffs(q->logical_mask) - 1); 355 356 return 0; 357 358 err_sched: 359 drm_sched_fini(&exl->sched); 360 err_free: 361 kfree(exl); 362 return err; 363 } 364 365 static void execlist_exec_queue_fini_async(struct work_struct *w) 366 { 367 struct xe_execlist_exec_queue *ee = 368 container_of(w, struct xe_execlist_exec_queue, fini_async); 369 struct xe_exec_queue *q = ee->q; 370 struct xe_execlist_exec_queue *exl = q->execlist; 371 struct xe_device *xe = gt_to_xe(q->gt); 372 unsigned long flags; 373 374 xe_assert(xe, !xe_device_uc_enabled(xe)); 375 376 spin_lock_irqsave(&exl->port->lock, flags); 377 if (WARN_ON(exl->active_priority != XE_EXEC_QUEUE_PRIORITY_UNSET)) 378 list_del(&exl->active_link); 379 spin_unlock_irqrestore(&exl->port->lock, flags); 380 381 drm_sched_entity_fini(&exl->entity); 382 drm_sched_fini(&exl->sched); 383 kfree(exl); 384 385 xe_exec_queue_fini(q); 386 } 387 388 static void execlist_exec_queue_kill(struct xe_exec_queue *q) 389 { 390 /* NIY */ 391 } 392 393 static void execlist_exec_queue_fini(struct xe_exec_queue *q) 394 { 395 INIT_WORK(&q->execlist->fini_async, execlist_exec_queue_fini_async); 396 queue_work(system_unbound_wq, &q->execlist->fini_async); 397 } 398 399 static int execlist_exec_queue_set_priority(struct xe_exec_queue *q, 400 enum xe_exec_queue_priority priority) 401 { 402 /* NIY */ 403 return 0; 404 } 405 406 static int execlist_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) 407 { 408 /* NIY */ 409 return 0; 410 } 411 412 static int execlist_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, 413 u32 preempt_timeout_us) 414 { 415 /* NIY */ 416 return 0; 417 } 418 419 static int execlist_exec_queue_suspend(struct xe_exec_queue *q) 420 { 421 /* NIY */ 422 return 0; 423 } 424 425 static void execlist_exec_queue_suspend_wait(struct xe_exec_queue *q) 426 427 { 428 /* NIY */ 429 } 430 431 static void execlist_exec_queue_resume(struct xe_exec_queue *q) 432 { 433 /* NIY */ 434 } 435 436 static bool execlist_exec_queue_reset_status(struct xe_exec_queue *q) 437 { 438 /* NIY */ 439 return false; 440 } 441 442 static const struct xe_exec_queue_ops execlist_exec_queue_ops = { 443 .init = execlist_exec_queue_init, 444 .kill = execlist_exec_queue_kill, 445 .fini = execlist_exec_queue_fini, 446 .set_priority = execlist_exec_queue_set_priority, 447 .set_timeslice = execlist_exec_queue_set_timeslice, 448 .set_preempt_timeout = execlist_exec_queue_set_preempt_timeout, 449 .suspend = execlist_exec_queue_suspend, 450 .suspend_wait = execlist_exec_queue_suspend_wait, 451 .resume = execlist_exec_queue_resume, 452 .reset_status = execlist_exec_queue_reset_status, 453 }; 454 455 int xe_execlist_init(struct xe_gt *gt) 456 { 457 /* GuC submission enabled, nothing to do */ 458 if (xe_device_uc_enabled(gt_to_xe(gt))) 459 return 0; 460 461 gt->exec_queue_ops = &execlist_exec_queue_ops; 462 463 return 0; 464 } 465