1 /* 2 * Copyright © 2008-2010 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * Zou Nan hai <nanhai.zou@intel.com> 26 * Xiang Hai hao<haihao.xiang@intel.com> 27 * 28 */ 29 30 #include <linux/log2.h> 31 32 #include <drm/i915_drm.h> 33 34 #include "gem/i915_gem_context.h" 35 36 #include "gen6_ppgtt.h" 37 #include "i915_drv.h" 38 #include "i915_trace.h" 39 #include "intel_context.h" 40 #include "intel_gt.h" 41 #include "intel_gt_irq.h" 42 #include "intel_gt_pm_irq.h" 43 #include "intel_reset.h" 44 #include "intel_ring.h" 45 #include "intel_workarounds.h" 46 47 /* Rough estimate of the typical request size, performing a flush, 48 * set-context and then emitting the batch. 49 */ 50 #define LEGACY_REQUEST_SIZE 200 51 52 static int 53 gen2_render_ring_flush(struct i915_request *rq, u32 mode) 54 { 55 unsigned int num_store_dw; 56 u32 cmd, *cs; 57 58 cmd = MI_FLUSH; 59 num_store_dw = 0; 60 if (mode & EMIT_INVALIDATE) 61 cmd |= MI_READ_FLUSH; 62 if (mode & EMIT_FLUSH) 63 num_store_dw = 4; 64 65 cs = intel_ring_begin(rq, 2 + 3 * num_store_dw); 66 if (IS_ERR(cs)) 67 return PTR_ERR(cs); 68 69 *cs++ = cmd; 70 while (num_store_dw--) { 71 *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 72 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 73 INTEL_GT_SCRATCH_FIELD_DEFAULT); 74 *cs++ = 0; 75 } 76 *cs++ = MI_FLUSH | MI_NO_WRITE_FLUSH; 77 78 intel_ring_advance(rq, cs); 79 80 return 0; 81 } 82 83 static int 84 gen4_render_ring_flush(struct i915_request *rq, u32 mode) 85 { 86 u32 cmd, *cs; 87 int i; 88 89 /* 90 * read/write caches: 91 * 92 * I915_GEM_DOMAIN_RENDER is always invalidated, but is 93 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is 94 * also flushed at 2d versus 3d pipeline switches. 95 * 96 * read-only caches: 97 * 98 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if 99 * MI_READ_FLUSH is set, and is always flushed on 965. 100 * 101 * I915_GEM_DOMAIN_COMMAND may not exist? 102 * 103 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is 104 * invalidated when MI_EXE_FLUSH is set. 105 * 106 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is 107 * invalidated with every MI_FLUSH. 108 * 109 * TLBs: 110 * 111 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND 112 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and 113 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER 114 * are flushed at any MI_FLUSH. 115 */ 116 117 cmd = MI_FLUSH; 118 if (mode & EMIT_INVALIDATE) { 119 cmd |= MI_EXE_FLUSH; 120 if (IS_G4X(rq->i915) || IS_GEN(rq->i915, 5)) 121 cmd |= MI_INVALIDATE_ISP; 122 } 123 124 i = 2; 125 if (mode & EMIT_INVALIDATE) 126 i += 20; 127 128 cs = intel_ring_begin(rq, i); 129 if (IS_ERR(cs)) 130 return PTR_ERR(cs); 131 132 *cs++ = cmd; 133 134 /* 135 * A random delay to let the CS invalidate take effect? Without this 136 * delay, the GPU relocation path fails as the CS does not see 137 * the updated contents. Just as important, if we apply the flushes 138 * to the EMIT_FLUSH branch (i.e. immediately after the relocation 139 * write and before the invalidate on the next batch), the relocations 140 * still fail. This implies that is a delay following invalidation 141 * that is required to reset the caches as opposed to a delay to 142 * ensure the memory is written. 143 */ 144 if (mode & EMIT_INVALIDATE) { 145 *cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE; 146 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 147 INTEL_GT_SCRATCH_FIELD_DEFAULT) | 148 PIPE_CONTROL_GLOBAL_GTT; 149 *cs++ = 0; 150 *cs++ = 0; 151 152 for (i = 0; i < 12; i++) 153 *cs++ = MI_FLUSH; 154 155 *cs++ = GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE; 156 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 157 INTEL_GT_SCRATCH_FIELD_DEFAULT) | 158 PIPE_CONTROL_GLOBAL_GTT; 159 *cs++ = 0; 160 *cs++ = 0; 161 } 162 163 *cs++ = cmd; 164 165 intel_ring_advance(rq, cs); 166 167 return 0; 168 } 169 170 /* 171 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for 172 * implementing two workarounds on gen6. From section 1.4.7.1 173 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1: 174 * 175 * [DevSNB-C+{W/A}] Before any depth stall flush (including those 176 * produced by non-pipelined state commands), software needs to first 177 * send a PIPE_CONTROL with no bits set except Post-Sync Operation != 178 * 0. 179 * 180 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable 181 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required. 182 * 183 * And the workaround for these two requires this workaround first: 184 * 185 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent 186 * BEFORE the pipe-control with a post-sync op and no write-cache 187 * flushes. 188 * 189 * And this last workaround is tricky because of the requirements on 190 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM 191 * volume 2 part 1: 192 * 193 * "1 of the following must also be set: 194 * - Render Target Cache Flush Enable ([12] of DW1) 195 * - Depth Cache Flush Enable ([0] of DW1) 196 * - Stall at Pixel Scoreboard ([1] of DW1) 197 * - Depth Stall ([13] of DW1) 198 * - Post-Sync Operation ([13] of DW1) 199 * - Notify Enable ([8] of DW1)" 200 * 201 * The cache flushes require the workaround flush that triggered this 202 * one, so we can't use it. Depth stall would trigger the same. 203 * Post-sync nonzero is what triggered this second workaround, so we 204 * can't use that one either. Notify enable is IRQs, which aren't 205 * really our business. That leaves only stall at scoreboard. 206 */ 207 static int 208 gen6_emit_post_sync_nonzero_flush(struct i915_request *rq) 209 { 210 u32 scratch_addr = 211 intel_gt_scratch_offset(rq->engine->gt, 212 INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH); 213 u32 *cs; 214 215 cs = intel_ring_begin(rq, 6); 216 if (IS_ERR(cs)) 217 return PTR_ERR(cs); 218 219 *cs++ = GFX_OP_PIPE_CONTROL(5); 220 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; 221 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; 222 *cs++ = 0; /* low dword */ 223 *cs++ = 0; /* high dword */ 224 *cs++ = MI_NOOP; 225 intel_ring_advance(rq, cs); 226 227 cs = intel_ring_begin(rq, 6); 228 if (IS_ERR(cs)) 229 return PTR_ERR(cs); 230 231 *cs++ = GFX_OP_PIPE_CONTROL(5); 232 *cs++ = PIPE_CONTROL_QW_WRITE; 233 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; 234 *cs++ = 0; 235 *cs++ = 0; 236 *cs++ = MI_NOOP; 237 intel_ring_advance(rq, cs); 238 239 return 0; 240 } 241 242 static int 243 gen6_render_ring_flush(struct i915_request *rq, u32 mode) 244 { 245 u32 scratch_addr = 246 intel_gt_scratch_offset(rq->engine->gt, 247 INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH); 248 u32 *cs, flags = 0; 249 int ret; 250 251 /* Force SNB workarounds for PIPE_CONTROL flushes */ 252 ret = gen6_emit_post_sync_nonzero_flush(rq); 253 if (ret) 254 return ret; 255 256 /* Just flush everything. Experiments have shown that reducing the 257 * number of bits based on the write domains has little performance 258 * impact. 259 */ 260 if (mode & EMIT_FLUSH) { 261 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; 262 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; 263 /* 264 * Ensure that any following seqno writes only happen 265 * when the render cache is indeed flushed. 266 */ 267 flags |= PIPE_CONTROL_CS_STALL; 268 } 269 if (mode & EMIT_INVALIDATE) { 270 flags |= PIPE_CONTROL_TLB_INVALIDATE; 271 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; 272 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 273 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 274 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; 275 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 276 /* 277 * TLB invalidate requires a post-sync write. 278 */ 279 flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL; 280 } 281 282 cs = intel_ring_begin(rq, 4); 283 if (IS_ERR(cs)) 284 return PTR_ERR(cs); 285 286 *cs++ = GFX_OP_PIPE_CONTROL(4); 287 *cs++ = flags; 288 *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT; 289 *cs++ = 0; 290 intel_ring_advance(rq, cs); 291 292 return 0; 293 } 294 295 static u32 *gen6_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 296 { 297 /* First we do the gen6_emit_post_sync_nonzero_flush w/a */ 298 *cs++ = GFX_OP_PIPE_CONTROL(4); 299 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; 300 *cs++ = 0; 301 *cs++ = 0; 302 303 *cs++ = GFX_OP_PIPE_CONTROL(4); 304 *cs++ = PIPE_CONTROL_QW_WRITE; 305 *cs++ = intel_gt_scratch_offset(rq->engine->gt, 306 INTEL_GT_SCRATCH_FIELD_DEFAULT) | 307 PIPE_CONTROL_GLOBAL_GTT; 308 *cs++ = 0; 309 310 /* Finally we can flush and with it emit the breadcrumb */ 311 *cs++ = GFX_OP_PIPE_CONTROL(4); 312 *cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 313 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 314 PIPE_CONTROL_DC_FLUSH_ENABLE | 315 PIPE_CONTROL_QW_WRITE | 316 PIPE_CONTROL_CS_STALL); 317 *cs++ = i915_request_active_timeline(rq)->hwsp_offset | 318 PIPE_CONTROL_GLOBAL_GTT; 319 *cs++ = rq->fence.seqno; 320 321 *cs++ = MI_USER_INTERRUPT; 322 *cs++ = MI_NOOP; 323 324 rq->tail = intel_ring_offset(rq, cs); 325 assert_ring_tail_valid(rq->ring, rq->tail); 326 327 return cs; 328 } 329 330 static int 331 gen7_render_ring_cs_stall_wa(struct i915_request *rq) 332 { 333 u32 *cs; 334 335 cs = intel_ring_begin(rq, 4); 336 if (IS_ERR(cs)) 337 return PTR_ERR(cs); 338 339 *cs++ = GFX_OP_PIPE_CONTROL(4); 340 *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD; 341 *cs++ = 0; 342 *cs++ = 0; 343 intel_ring_advance(rq, cs); 344 345 return 0; 346 } 347 348 static int 349 gen7_render_ring_flush(struct i915_request *rq, u32 mode) 350 { 351 u32 scratch_addr = 352 intel_gt_scratch_offset(rq->engine->gt, 353 INTEL_GT_SCRATCH_FIELD_RENDER_FLUSH); 354 u32 *cs, flags = 0; 355 356 /* 357 * Ensure that any following seqno writes only happen when the render 358 * cache is indeed flushed. 359 * 360 * Workaround: 4th PIPE_CONTROL command (except the ones with only 361 * read-cache invalidate bits set) must have the CS_STALL bit set. We 362 * don't try to be clever and just set it unconditionally. 363 */ 364 flags |= PIPE_CONTROL_CS_STALL; 365 366 /* 367 * CS_STALL suggests at least a post-sync write. 368 */ 369 flags |= PIPE_CONTROL_QW_WRITE; 370 flags |= PIPE_CONTROL_GLOBAL_GTT_IVB; 371 372 /* Just flush everything. Experiments have shown that reducing the 373 * number of bits based on the write domains has little performance 374 * impact. 375 */ 376 if (mode & EMIT_FLUSH) { 377 flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH; 378 flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH; 379 flags |= PIPE_CONTROL_DC_FLUSH_ENABLE; 380 flags |= PIPE_CONTROL_FLUSH_ENABLE; 381 } 382 if (mode & EMIT_INVALIDATE) { 383 flags |= PIPE_CONTROL_TLB_INVALIDATE; 384 flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE; 385 flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE; 386 flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE; 387 flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE; 388 flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE; 389 flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR; 390 391 /* Workaround: we must issue a pipe_control with CS-stall bit 392 * set before a pipe_control command that has the state cache 393 * invalidate bit set. */ 394 gen7_render_ring_cs_stall_wa(rq); 395 } 396 397 cs = intel_ring_begin(rq, 4); 398 if (IS_ERR(cs)) 399 return PTR_ERR(cs); 400 401 *cs++ = GFX_OP_PIPE_CONTROL(4); 402 *cs++ = flags; 403 *cs++ = scratch_addr; 404 *cs++ = 0; 405 intel_ring_advance(rq, cs); 406 407 return 0; 408 } 409 410 static u32 *gen7_rcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 411 { 412 *cs++ = GFX_OP_PIPE_CONTROL(4); 413 *cs++ = (PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 414 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 415 PIPE_CONTROL_DC_FLUSH_ENABLE | 416 PIPE_CONTROL_FLUSH_ENABLE | 417 PIPE_CONTROL_QW_WRITE | 418 PIPE_CONTROL_GLOBAL_GTT_IVB | 419 PIPE_CONTROL_CS_STALL); 420 *cs++ = i915_request_active_timeline(rq)->hwsp_offset; 421 *cs++ = rq->fence.seqno; 422 423 *cs++ = MI_USER_INTERRUPT; 424 *cs++ = MI_NOOP; 425 426 rq->tail = intel_ring_offset(rq, cs); 427 assert_ring_tail_valid(rq->ring, rq->tail); 428 429 return cs; 430 } 431 432 static u32 *gen6_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 433 { 434 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 435 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 436 437 *cs++ = MI_FLUSH_DW | MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; 438 *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT; 439 *cs++ = rq->fence.seqno; 440 441 *cs++ = MI_USER_INTERRUPT; 442 443 rq->tail = intel_ring_offset(rq, cs); 444 assert_ring_tail_valid(rq->ring, rq->tail); 445 446 return cs; 447 } 448 449 #define GEN7_XCS_WA 32 450 static u32 *gen7_xcs_emit_breadcrumb(struct i915_request *rq, u32 *cs) 451 { 452 int i; 453 454 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 455 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 456 457 *cs++ = MI_FLUSH_DW | MI_INVALIDATE_TLB | 458 MI_FLUSH_DW_OP_STOREDW | MI_FLUSH_DW_STORE_INDEX; 459 *cs++ = I915_GEM_HWS_SEQNO_ADDR | MI_FLUSH_DW_USE_GTT; 460 *cs++ = rq->fence.seqno; 461 462 for (i = 0; i < GEN7_XCS_WA; i++) { 463 *cs++ = MI_STORE_DWORD_INDEX; 464 *cs++ = I915_GEM_HWS_SEQNO_ADDR; 465 *cs++ = rq->fence.seqno; 466 } 467 468 *cs++ = MI_FLUSH_DW; 469 *cs++ = 0; 470 *cs++ = 0; 471 472 *cs++ = MI_USER_INTERRUPT; 473 *cs++ = MI_NOOP; 474 475 rq->tail = intel_ring_offset(rq, cs); 476 assert_ring_tail_valid(rq->ring, rq->tail); 477 478 return cs; 479 } 480 #undef GEN7_XCS_WA 481 482 static void set_hwstam(struct intel_engine_cs *engine, u32 mask) 483 { 484 /* 485 * Keep the render interrupt unmasked as this papers over 486 * lost interrupts following a reset. 487 */ 488 if (engine->class == RENDER_CLASS) { 489 if (INTEL_GEN(engine->i915) >= 6) 490 mask &= ~BIT(0); 491 else 492 mask &= ~I915_USER_INTERRUPT; 493 } 494 495 intel_engine_set_hwsp_writemask(engine, mask); 496 } 497 498 static void set_hws_pga(struct intel_engine_cs *engine, phys_addr_t phys) 499 { 500 u32 addr; 501 502 addr = lower_32_bits(phys); 503 if (INTEL_GEN(engine->i915) >= 4) 504 addr |= (phys >> 28) & 0xf0; 505 506 intel_uncore_write(engine->uncore, HWS_PGA, addr); 507 } 508 509 static struct page *status_page(struct intel_engine_cs *engine) 510 { 511 struct drm_i915_gem_object *obj = engine->status_page.vma->obj; 512 513 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj)); 514 return sg_page(obj->mm.pages->sgl); 515 } 516 517 static void ring_setup_phys_status_page(struct intel_engine_cs *engine) 518 { 519 set_hws_pga(engine, PFN_PHYS(page_to_pfn(status_page(engine)))); 520 set_hwstam(engine, ~0u); 521 } 522 523 static void set_hwsp(struct intel_engine_cs *engine, u32 offset) 524 { 525 i915_reg_t hwsp; 526 527 /* 528 * The ring status page addresses are no longer next to the rest of 529 * the ring registers as of gen7. 530 */ 531 if (IS_GEN(engine->i915, 7)) { 532 switch (engine->id) { 533 /* 534 * No more rings exist on Gen7. Default case is only to shut up 535 * gcc switch check warning. 536 */ 537 default: 538 GEM_BUG_ON(engine->id); 539 /* fallthrough */ 540 case RCS0: 541 hwsp = RENDER_HWS_PGA_GEN7; 542 break; 543 case BCS0: 544 hwsp = BLT_HWS_PGA_GEN7; 545 break; 546 case VCS0: 547 hwsp = BSD_HWS_PGA_GEN7; 548 break; 549 case VECS0: 550 hwsp = VEBOX_HWS_PGA_GEN7; 551 break; 552 } 553 } else if (IS_GEN(engine->i915, 6)) { 554 hwsp = RING_HWS_PGA_GEN6(engine->mmio_base); 555 } else { 556 hwsp = RING_HWS_PGA(engine->mmio_base); 557 } 558 559 intel_uncore_write(engine->uncore, hwsp, offset); 560 intel_uncore_posting_read(engine->uncore, hwsp); 561 } 562 563 static void flush_cs_tlb(struct intel_engine_cs *engine) 564 { 565 struct drm_i915_private *dev_priv = engine->i915; 566 567 if (!IS_GEN_RANGE(dev_priv, 6, 7)) 568 return; 569 570 /* ring should be idle before issuing a sync flush*/ 571 drm_WARN_ON(&dev_priv->drm, 572 (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 573 574 ENGINE_WRITE(engine, RING_INSTPM, 575 _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE | 576 INSTPM_SYNC_FLUSH)); 577 if (intel_wait_for_register(engine->uncore, 578 RING_INSTPM(engine->mmio_base), 579 INSTPM_SYNC_FLUSH, 0, 580 1000)) 581 DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n", 582 engine->name); 583 } 584 585 static void ring_setup_status_page(struct intel_engine_cs *engine) 586 { 587 set_hwsp(engine, i915_ggtt_offset(engine->status_page.vma)); 588 set_hwstam(engine, ~0u); 589 590 flush_cs_tlb(engine); 591 } 592 593 static bool stop_ring(struct intel_engine_cs *engine) 594 { 595 struct drm_i915_private *dev_priv = engine->i915; 596 597 if (INTEL_GEN(dev_priv) > 2) { 598 ENGINE_WRITE(engine, 599 RING_MI_MODE, _MASKED_BIT_ENABLE(STOP_RING)); 600 if (intel_wait_for_register(engine->uncore, 601 RING_MI_MODE(engine->mmio_base), 602 MODE_IDLE, 603 MODE_IDLE, 604 1000)) { 605 DRM_ERROR("%s : timed out trying to stop ring\n", 606 engine->name); 607 608 /* 609 * Sometimes we observe that the idle flag is not 610 * set even though the ring is empty. So double 611 * check before giving up. 612 */ 613 if (ENGINE_READ(engine, RING_HEAD) != 614 ENGINE_READ(engine, RING_TAIL)) 615 return false; 616 } 617 } 618 619 ENGINE_WRITE(engine, RING_HEAD, ENGINE_READ(engine, RING_TAIL)); 620 621 ENGINE_WRITE(engine, RING_HEAD, 0); 622 ENGINE_WRITE(engine, RING_TAIL, 0); 623 624 /* The ring must be empty before it is disabled */ 625 ENGINE_WRITE(engine, RING_CTL, 0); 626 627 return (ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) == 0; 628 } 629 630 static struct i915_address_space *vm_alias(struct i915_address_space *vm) 631 { 632 if (i915_is_ggtt(vm)) 633 vm = &i915_vm_to_ggtt(vm)->alias->vm; 634 635 return vm; 636 } 637 638 static void set_pp_dir(struct intel_engine_cs *engine) 639 { 640 struct i915_address_space *vm = vm_alias(engine->gt->vm); 641 642 if (vm) { 643 struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 644 645 ENGINE_WRITE(engine, RING_PP_DIR_DCLV, PP_DIR_DCLV_2G); 646 ENGINE_WRITE(engine, RING_PP_DIR_BASE, 647 px_base(ppgtt->pd)->ggtt_offset << 10); 648 } 649 } 650 651 static int xcs_resume(struct intel_engine_cs *engine) 652 { 653 struct drm_i915_private *dev_priv = engine->i915; 654 struct intel_ring *ring = engine->legacy.ring; 655 int ret = 0; 656 657 ENGINE_TRACE(engine, "ring:{HEAD:%04x, TAIL:%04x}\n", 658 ring->head, ring->tail); 659 660 intel_uncore_forcewake_get(engine->uncore, FORCEWAKE_ALL); 661 662 /* WaClearRingBufHeadRegAtInit:ctg,elk */ 663 if (!stop_ring(engine)) { 664 /* G45 ring initialization often fails to reset head to zero */ 665 DRM_DEBUG_DRIVER("%s head not reset to zero " 666 "ctl %08x head %08x tail %08x start %08x\n", 667 engine->name, 668 ENGINE_READ(engine, RING_CTL), 669 ENGINE_READ(engine, RING_HEAD), 670 ENGINE_READ(engine, RING_TAIL), 671 ENGINE_READ(engine, RING_START)); 672 673 if (!stop_ring(engine)) { 674 DRM_ERROR("failed to set %s head to zero " 675 "ctl %08x head %08x tail %08x start %08x\n", 676 engine->name, 677 ENGINE_READ(engine, RING_CTL), 678 ENGINE_READ(engine, RING_HEAD), 679 ENGINE_READ(engine, RING_TAIL), 680 ENGINE_READ(engine, RING_START)); 681 ret = -EIO; 682 goto out; 683 } 684 } 685 686 if (HWS_NEEDS_PHYSICAL(dev_priv)) 687 ring_setup_phys_status_page(engine); 688 else 689 ring_setup_status_page(engine); 690 691 intel_engine_reset_breadcrumbs(engine); 692 693 /* Enforce ordering by reading HEAD register back */ 694 ENGINE_POSTING_READ(engine, RING_HEAD); 695 696 /* 697 * Initialize the ring. This must happen _after_ we've cleared the ring 698 * registers with the above sequence (the readback of the HEAD registers 699 * also enforces ordering), otherwise the hw might lose the new ring 700 * register values. 701 */ 702 ENGINE_WRITE(engine, RING_START, i915_ggtt_offset(ring->vma)); 703 704 /* Check that the ring offsets point within the ring! */ 705 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->head)); 706 GEM_BUG_ON(!intel_ring_offset_valid(ring, ring->tail)); 707 intel_ring_update_space(ring); 708 709 set_pp_dir(engine); 710 711 /* First wake the ring up to an empty/idle ring */ 712 ENGINE_WRITE(engine, RING_HEAD, ring->head); 713 ENGINE_WRITE(engine, RING_TAIL, ring->head); 714 ENGINE_POSTING_READ(engine, RING_TAIL); 715 716 ENGINE_WRITE(engine, RING_CTL, RING_CTL_SIZE(ring->size) | RING_VALID); 717 718 /* If the head is still not zero, the ring is dead */ 719 if (intel_wait_for_register(engine->uncore, 720 RING_CTL(engine->mmio_base), 721 RING_VALID, RING_VALID, 722 50)) { 723 DRM_ERROR("%s initialization failed " 724 "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n", 725 engine->name, 726 ENGINE_READ(engine, RING_CTL), 727 ENGINE_READ(engine, RING_CTL) & RING_VALID, 728 ENGINE_READ(engine, RING_HEAD), ring->head, 729 ENGINE_READ(engine, RING_TAIL), ring->tail, 730 ENGINE_READ(engine, RING_START), 731 i915_ggtt_offset(ring->vma)); 732 ret = -EIO; 733 goto out; 734 } 735 736 if (INTEL_GEN(dev_priv) > 2) 737 ENGINE_WRITE(engine, 738 RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING)); 739 740 /* Now awake, let it get started */ 741 if (ring->tail != ring->head) { 742 ENGINE_WRITE(engine, RING_TAIL, ring->tail); 743 ENGINE_POSTING_READ(engine, RING_TAIL); 744 } 745 746 /* Papering over lost _interrupts_ immediately following the restart */ 747 intel_engine_signal_breadcrumbs(engine); 748 out: 749 intel_uncore_forcewake_put(engine->uncore, FORCEWAKE_ALL); 750 751 return ret; 752 } 753 754 static void reset_prepare(struct intel_engine_cs *engine) 755 { 756 struct intel_uncore *uncore = engine->uncore; 757 const u32 base = engine->mmio_base; 758 759 /* 760 * We stop engines, otherwise we might get failed reset and a 761 * dead gpu (on elk). Also as modern gpu as kbl can suffer 762 * from system hang if batchbuffer is progressing when 763 * the reset is issued, regardless of READY_TO_RESET ack. 764 * Thus assume it is best to stop engines on all gens 765 * where we have a gpu reset. 766 * 767 * WaKBLVECSSemaphoreWaitPoll:kbl (on ALL_ENGINES) 768 * 769 * WaMediaResetMainRingCleanup:ctg,elk (presumably) 770 * 771 * FIXME: Wa for more modern gens needs to be validated 772 */ 773 ENGINE_TRACE(engine, "\n"); 774 775 if (intel_engine_stop_cs(engine)) 776 ENGINE_TRACE(engine, "timed out on STOP_RING\n"); 777 778 intel_uncore_write_fw(uncore, 779 RING_HEAD(base), 780 intel_uncore_read_fw(uncore, RING_TAIL(base))); 781 intel_uncore_posting_read_fw(uncore, RING_HEAD(base)); /* paranoia */ 782 783 intel_uncore_write_fw(uncore, RING_HEAD(base), 0); 784 intel_uncore_write_fw(uncore, RING_TAIL(base), 0); 785 intel_uncore_posting_read_fw(uncore, RING_TAIL(base)); 786 787 /* The ring must be empty before it is disabled */ 788 intel_uncore_write_fw(uncore, RING_CTL(base), 0); 789 790 /* Check acts as a post */ 791 if (intel_uncore_read_fw(uncore, RING_HEAD(base))) 792 ENGINE_TRACE(engine, "ring head [%x] not parked\n", 793 intel_uncore_read_fw(uncore, RING_HEAD(base))); 794 } 795 796 static void reset_rewind(struct intel_engine_cs *engine, bool stalled) 797 { 798 struct i915_request *pos, *rq; 799 unsigned long flags; 800 u32 head; 801 802 rq = NULL; 803 spin_lock_irqsave(&engine->active.lock, flags); 804 list_for_each_entry(pos, &engine->active.requests, sched.link) { 805 if (!i915_request_completed(pos)) { 806 rq = pos; 807 break; 808 } 809 } 810 811 /* 812 * The guilty request will get skipped on a hung engine. 813 * 814 * Users of client default contexts do not rely on logical 815 * state preserved between batches so it is safe to execute 816 * queued requests following the hang. Non default contexts 817 * rely on preserved state, so skipping a batch loses the 818 * evolution of the state and it needs to be considered corrupted. 819 * Executing more queued batches on top of corrupted state is 820 * risky. But we take the risk by trying to advance through 821 * the queued requests in order to make the client behaviour 822 * more predictable around resets, by not throwing away random 823 * amount of batches it has prepared for execution. Sophisticated 824 * clients can use gem_reset_stats_ioctl and dma fence status 825 * (exported via sync_file info ioctl on explicit fences) to observe 826 * when it loses the context state and should rebuild accordingly. 827 * 828 * The context ban, and ultimately the client ban, mechanism are safety 829 * valves if client submission ends up resulting in nothing more than 830 * subsequent hangs. 831 */ 832 833 if (rq) { 834 /* 835 * Try to restore the logical GPU state to match the 836 * continuation of the request queue. If we skip the 837 * context/PD restore, then the next request may try to execute 838 * assuming that its context is valid and loaded on the GPU and 839 * so may try to access invalid memory, prompting repeated GPU 840 * hangs. 841 * 842 * If the request was guilty, we still restore the logical 843 * state in case the next request requires it (e.g. the 844 * aliasing ppgtt), but skip over the hung batch. 845 * 846 * If the request was innocent, we try to replay the request 847 * with the restored context. 848 */ 849 __i915_request_reset(rq, stalled); 850 851 GEM_BUG_ON(rq->ring != engine->legacy.ring); 852 head = rq->head; 853 } else { 854 head = engine->legacy.ring->tail; 855 } 856 engine->legacy.ring->head = intel_ring_wrap(engine->legacy.ring, head); 857 858 spin_unlock_irqrestore(&engine->active.lock, flags); 859 } 860 861 static void reset_finish(struct intel_engine_cs *engine) 862 { 863 } 864 865 static int rcs_resume(struct intel_engine_cs *engine) 866 { 867 struct drm_i915_private *i915 = engine->i915; 868 struct intel_uncore *uncore = engine->uncore; 869 870 /* 871 * Disable CONSTANT_BUFFER before it is loaded from the context 872 * image. For as it is loaded, it is executed and the stored 873 * address may no longer be valid, leading to a GPU hang. 874 * 875 * This imposes the requirement that userspace reload their 876 * CONSTANT_BUFFER on every batch, fortunately a requirement 877 * they are already accustomed to from before contexts were 878 * enabled. 879 */ 880 if (IS_GEN(i915, 4)) 881 intel_uncore_write(uncore, ECOSKPD, 882 _MASKED_BIT_ENABLE(ECO_CONSTANT_BUFFER_SR_DISABLE)); 883 884 if (IS_GEN_RANGE(i915, 6, 7)) 885 intel_uncore_write(uncore, INSTPM, 886 _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING)); 887 888 return xcs_resume(engine); 889 } 890 891 static void reset_cancel(struct intel_engine_cs *engine) 892 { 893 struct i915_request *request; 894 unsigned long flags; 895 896 spin_lock_irqsave(&engine->active.lock, flags); 897 898 /* Mark all submitted requests as skipped. */ 899 list_for_each_entry(request, &engine->active.requests, sched.link) { 900 if (!i915_request_signaled(request)) 901 dma_fence_set_error(&request->fence, -EIO); 902 903 i915_request_mark_complete(request); 904 } 905 906 /* Remaining _unready_ requests will be nop'ed when submitted */ 907 908 spin_unlock_irqrestore(&engine->active.lock, flags); 909 } 910 911 static void i9xx_submit_request(struct i915_request *request) 912 { 913 i915_request_submit(request); 914 wmb(); /* paranoid flush writes out of the WCB before mmio */ 915 916 ENGINE_WRITE(request->engine, RING_TAIL, 917 intel_ring_set_tail(request->ring, request->tail)); 918 } 919 920 static u32 *i9xx_emit_breadcrumb(struct i915_request *rq, u32 *cs) 921 { 922 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 923 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 924 925 *cs++ = MI_FLUSH; 926 927 *cs++ = MI_STORE_DWORD_INDEX; 928 *cs++ = I915_GEM_HWS_SEQNO_ADDR; 929 *cs++ = rq->fence.seqno; 930 931 *cs++ = MI_USER_INTERRUPT; 932 *cs++ = MI_NOOP; 933 934 rq->tail = intel_ring_offset(rq, cs); 935 assert_ring_tail_valid(rq->ring, rq->tail); 936 937 return cs; 938 } 939 940 #define GEN5_WA_STORES 8 /* must be at least 1! */ 941 static u32 *gen5_emit_breadcrumb(struct i915_request *rq, u32 *cs) 942 { 943 int i; 944 945 GEM_BUG_ON(i915_request_active_timeline(rq)->hwsp_ggtt != rq->engine->status_page.vma); 946 GEM_BUG_ON(offset_in_page(i915_request_active_timeline(rq)->hwsp_offset) != I915_GEM_HWS_SEQNO_ADDR); 947 948 *cs++ = MI_FLUSH; 949 950 BUILD_BUG_ON(GEN5_WA_STORES < 1); 951 for (i = 0; i < GEN5_WA_STORES; i++) { 952 *cs++ = MI_STORE_DWORD_INDEX; 953 *cs++ = I915_GEM_HWS_SEQNO_ADDR; 954 *cs++ = rq->fence.seqno; 955 } 956 957 *cs++ = MI_USER_INTERRUPT; 958 959 rq->tail = intel_ring_offset(rq, cs); 960 assert_ring_tail_valid(rq->ring, rq->tail); 961 962 return cs; 963 } 964 #undef GEN5_WA_STORES 965 966 static void 967 gen5_irq_enable(struct intel_engine_cs *engine) 968 { 969 gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask); 970 } 971 972 static void 973 gen5_irq_disable(struct intel_engine_cs *engine) 974 { 975 gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask); 976 } 977 978 static void 979 i9xx_irq_enable(struct intel_engine_cs *engine) 980 { 981 engine->i915->irq_mask &= ~engine->irq_enable_mask; 982 intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask); 983 intel_uncore_posting_read_fw(engine->uncore, GEN2_IMR); 984 } 985 986 static void 987 i9xx_irq_disable(struct intel_engine_cs *engine) 988 { 989 engine->i915->irq_mask |= engine->irq_enable_mask; 990 intel_uncore_write(engine->uncore, GEN2_IMR, engine->i915->irq_mask); 991 } 992 993 static void 994 i8xx_irq_enable(struct intel_engine_cs *engine) 995 { 996 struct drm_i915_private *i915 = engine->i915; 997 998 i915->irq_mask &= ~engine->irq_enable_mask; 999 intel_uncore_write16(&i915->uncore, GEN2_IMR, i915->irq_mask); 1000 ENGINE_POSTING_READ16(engine, RING_IMR); 1001 } 1002 1003 static void 1004 i8xx_irq_disable(struct intel_engine_cs *engine) 1005 { 1006 struct drm_i915_private *i915 = engine->i915; 1007 1008 i915->irq_mask |= engine->irq_enable_mask; 1009 intel_uncore_write16(&i915->uncore, GEN2_IMR, i915->irq_mask); 1010 } 1011 1012 static int 1013 bsd_ring_flush(struct i915_request *rq, u32 mode) 1014 { 1015 u32 *cs; 1016 1017 cs = intel_ring_begin(rq, 2); 1018 if (IS_ERR(cs)) 1019 return PTR_ERR(cs); 1020 1021 *cs++ = MI_FLUSH; 1022 *cs++ = MI_NOOP; 1023 intel_ring_advance(rq, cs); 1024 return 0; 1025 } 1026 1027 static void 1028 gen6_irq_enable(struct intel_engine_cs *engine) 1029 { 1030 ENGINE_WRITE(engine, RING_IMR, 1031 ~(engine->irq_enable_mask | engine->irq_keep_mask)); 1032 1033 /* Flush/delay to ensure the RING_IMR is active before the GT IMR */ 1034 ENGINE_POSTING_READ(engine, RING_IMR); 1035 1036 gen5_gt_enable_irq(engine->gt, engine->irq_enable_mask); 1037 } 1038 1039 static void 1040 gen6_irq_disable(struct intel_engine_cs *engine) 1041 { 1042 ENGINE_WRITE(engine, RING_IMR, ~engine->irq_keep_mask); 1043 gen5_gt_disable_irq(engine->gt, engine->irq_enable_mask); 1044 } 1045 1046 static void 1047 hsw_vebox_irq_enable(struct intel_engine_cs *engine) 1048 { 1049 ENGINE_WRITE(engine, RING_IMR, ~engine->irq_enable_mask); 1050 1051 /* Flush/delay to ensure the RING_IMR is active before the GT IMR */ 1052 ENGINE_POSTING_READ(engine, RING_IMR); 1053 1054 gen6_gt_pm_unmask_irq(engine->gt, engine->irq_enable_mask); 1055 } 1056 1057 static void 1058 hsw_vebox_irq_disable(struct intel_engine_cs *engine) 1059 { 1060 ENGINE_WRITE(engine, RING_IMR, ~0); 1061 gen6_gt_pm_mask_irq(engine->gt, engine->irq_enable_mask); 1062 } 1063 1064 static int 1065 i965_emit_bb_start(struct i915_request *rq, 1066 u64 offset, u32 length, 1067 unsigned int dispatch_flags) 1068 { 1069 u32 *cs; 1070 1071 cs = intel_ring_begin(rq, 2); 1072 if (IS_ERR(cs)) 1073 return PTR_ERR(cs); 1074 1075 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | (dispatch_flags & 1076 I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965); 1077 *cs++ = offset; 1078 intel_ring_advance(rq, cs); 1079 1080 return 0; 1081 } 1082 1083 /* Just userspace ABI convention to limit the wa batch bo to a resonable size */ 1084 #define I830_BATCH_LIMIT SZ_256K 1085 #define I830_TLB_ENTRIES (2) 1086 #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT) 1087 static int 1088 i830_emit_bb_start(struct i915_request *rq, 1089 u64 offset, u32 len, 1090 unsigned int dispatch_flags) 1091 { 1092 u32 *cs, cs_offset = 1093 intel_gt_scratch_offset(rq->engine->gt, 1094 INTEL_GT_SCRATCH_FIELD_DEFAULT); 1095 1096 GEM_BUG_ON(rq->engine->gt->scratch->size < I830_WA_SIZE); 1097 1098 cs = intel_ring_begin(rq, 6); 1099 if (IS_ERR(cs)) 1100 return PTR_ERR(cs); 1101 1102 /* Evict the invalid PTE TLBs */ 1103 *cs++ = COLOR_BLT_CMD | BLT_WRITE_RGBA; 1104 *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096; 1105 *cs++ = I830_TLB_ENTRIES << 16 | 4; /* load each page */ 1106 *cs++ = cs_offset; 1107 *cs++ = 0xdeadbeef; 1108 *cs++ = MI_NOOP; 1109 intel_ring_advance(rq, cs); 1110 1111 if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) { 1112 if (len > I830_BATCH_LIMIT) 1113 return -ENOSPC; 1114 1115 cs = intel_ring_begin(rq, 6 + 2); 1116 if (IS_ERR(cs)) 1117 return PTR_ERR(cs); 1118 1119 /* Blit the batch (which has now all relocs applied) to the 1120 * stable batch scratch bo area (so that the CS never 1121 * stumbles over its tlb invalidation bug) ... 1122 */ 1123 *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA | (6 - 2); 1124 *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096; 1125 *cs++ = DIV_ROUND_UP(len, 4096) << 16 | 4096; 1126 *cs++ = cs_offset; 1127 *cs++ = 4096; 1128 *cs++ = offset; 1129 1130 *cs++ = MI_FLUSH; 1131 *cs++ = MI_NOOP; 1132 intel_ring_advance(rq, cs); 1133 1134 /* ... and execute it. */ 1135 offset = cs_offset; 1136 } 1137 1138 cs = intel_ring_begin(rq, 2); 1139 if (IS_ERR(cs)) 1140 return PTR_ERR(cs); 1141 1142 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 1143 *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 : 1144 MI_BATCH_NON_SECURE); 1145 intel_ring_advance(rq, cs); 1146 1147 return 0; 1148 } 1149 1150 static int 1151 i915_emit_bb_start(struct i915_request *rq, 1152 u64 offset, u32 len, 1153 unsigned int dispatch_flags) 1154 { 1155 u32 *cs; 1156 1157 cs = intel_ring_begin(rq, 2); 1158 if (IS_ERR(cs)) 1159 return PTR_ERR(cs); 1160 1161 *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 1162 *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 : 1163 MI_BATCH_NON_SECURE); 1164 intel_ring_advance(rq, cs); 1165 1166 return 0; 1167 } 1168 1169 static void __ring_context_fini(struct intel_context *ce) 1170 { 1171 i915_vma_put(ce->state); 1172 } 1173 1174 static void ring_context_destroy(struct kref *ref) 1175 { 1176 struct intel_context *ce = container_of(ref, typeof(*ce), ref); 1177 1178 GEM_BUG_ON(intel_context_is_pinned(ce)); 1179 1180 if (ce->state) 1181 __ring_context_fini(ce); 1182 1183 intel_context_fini(ce); 1184 intel_context_free(ce); 1185 } 1186 1187 static int __context_pin_ppgtt(struct intel_context *ce) 1188 { 1189 struct i915_address_space *vm; 1190 int err = 0; 1191 1192 vm = vm_alias(ce->vm); 1193 if (vm) 1194 err = gen6_ppgtt_pin(i915_vm_to_ppgtt((vm))); 1195 1196 return err; 1197 } 1198 1199 static void __context_unpin_ppgtt(struct intel_context *ce) 1200 { 1201 struct i915_address_space *vm; 1202 1203 vm = vm_alias(ce->vm); 1204 if (vm) 1205 gen6_ppgtt_unpin(i915_vm_to_ppgtt(vm)); 1206 } 1207 1208 static void ring_context_unpin(struct intel_context *ce) 1209 { 1210 __context_unpin_ppgtt(ce); 1211 } 1212 1213 static struct i915_vma * 1214 alloc_context_vma(struct intel_engine_cs *engine) 1215 { 1216 struct drm_i915_private *i915 = engine->i915; 1217 struct drm_i915_gem_object *obj; 1218 struct i915_vma *vma; 1219 int err; 1220 1221 obj = i915_gem_object_create_shmem(i915, engine->context_size); 1222 if (IS_ERR(obj)) 1223 return ERR_CAST(obj); 1224 1225 /* 1226 * Try to make the context utilize L3 as well as LLC. 1227 * 1228 * On VLV we don't have L3 controls in the PTEs so we 1229 * shouldn't touch the cache level, especially as that 1230 * would make the object snooped which might have a 1231 * negative performance impact. 1232 * 1233 * Snooping is required on non-llc platforms in execlist 1234 * mode, but since all GGTT accesses use PAT entry 0 we 1235 * get snooping anyway regardless of cache_level. 1236 * 1237 * This is only applicable for Ivy Bridge devices since 1238 * later platforms don't have L3 control bits in the PTE. 1239 */ 1240 if (IS_IVYBRIDGE(i915)) 1241 i915_gem_object_set_cache_coherency(obj, I915_CACHE_L3_LLC); 1242 1243 if (engine->default_state) { 1244 void *defaults, *vaddr; 1245 1246 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB); 1247 if (IS_ERR(vaddr)) { 1248 err = PTR_ERR(vaddr); 1249 goto err_obj; 1250 } 1251 1252 defaults = i915_gem_object_pin_map(engine->default_state, 1253 I915_MAP_WB); 1254 if (IS_ERR(defaults)) { 1255 err = PTR_ERR(defaults); 1256 goto err_map; 1257 } 1258 1259 memcpy(vaddr, defaults, engine->context_size); 1260 i915_gem_object_unpin_map(engine->default_state); 1261 1262 i915_gem_object_flush_map(obj); 1263 i915_gem_object_unpin_map(obj); 1264 } 1265 1266 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL); 1267 if (IS_ERR(vma)) { 1268 err = PTR_ERR(vma); 1269 goto err_obj; 1270 } 1271 1272 return vma; 1273 1274 err_map: 1275 i915_gem_object_unpin_map(obj); 1276 err_obj: 1277 i915_gem_object_put(obj); 1278 return ERR_PTR(err); 1279 } 1280 1281 static int ring_context_alloc(struct intel_context *ce) 1282 { 1283 struct intel_engine_cs *engine = ce->engine; 1284 1285 /* One ringbuffer to rule them all */ 1286 GEM_BUG_ON(!engine->legacy.ring); 1287 ce->ring = engine->legacy.ring; 1288 ce->timeline = intel_timeline_get(engine->legacy.timeline); 1289 1290 GEM_BUG_ON(ce->state); 1291 if (engine->context_size) { 1292 struct i915_vma *vma; 1293 1294 vma = alloc_context_vma(engine); 1295 if (IS_ERR(vma)) 1296 return PTR_ERR(vma); 1297 1298 ce->state = vma; 1299 if (engine->default_state) 1300 __set_bit(CONTEXT_VALID_BIT, &ce->flags); 1301 } 1302 1303 return 0; 1304 } 1305 1306 static int ring_context_pin(struct intel_context *ce) 1307 { 1308 return __context_pin_ppgtt(ce); 1309 } 1310 1311 static void ring_context_reset(struct intel_context *ce) 1312 { 1313 intel_ring_reset(ce->ring, ce->ring->emit); 1314 } 1315 1316 static const struct intel_context_ops ring_context_ops = { 1317 .alloc = ring_context_alloc, 1318 1319 .pin = ring_context_pin, 1320 .unpin = ring_context_unpin, 1321 1322 .enter = intel_context_enter_engine, 1323 .exit = intel_context_exit_engine, 1324 1325 .reset = ring_context_reset, 1326 .destroy = ring_context_destroy, 1327 }; 1328 1329 static int load_pd_dir(struct i915_request *rq, 1330 const struct i915_ppgtt *ppgtt, 1331 u32 valid) 1332 { 1333 const struct intel_engine_cs * const engine = rq->engine; 1334 u32 *cs; 1335 1336 cs = intel_ring_begin(rq, 12); 1337 if (IS_ERR(cs)) 1338 return PTR_ERR(cs); 1339 1340 *cs++ = MI_LOAD_REGISTER_IMM(1); 1341 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_DCLV(engine->mmio_base)); 1342 *cs++ = valid; 1343 1344 *cs++ = MI_LOAD_REGISTER_IMM(1); 1345 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 1346 *cs++ = px_base(ppgtt->pd)->ggtt_offset << 10; 1347 1348 /* Stall until the page table load is complete? */ 1349 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 1350 *cs++ = i915_mmio_reg_offset(RING_PP_DIR_BASE(engine->mmio_base)); 1351 *cs++ = intel_gt_scratch_offset(engine->gt, 1352 INTEL_GT_SCRATCH_FIELD_DEFAULT); 1353 1354 *cs++ = MI_LOAD_REGISTER_IMM(1); 1355 *cs++ = i915_mmio_reg_offset(RING_INSTPM(engine->mmio_base)); 1356 *cs++ = _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE); 1357 1358 intel_ring_advance(rq, cs); 1359 1360 return rq->engine->emit_flush(rq, EMIT_FLUSH); 1361 } 1362 1363 static inline int mi_set_context(struct i915_request *rq, u32 flags) 1364 { 1365 struct drm_i915_private *i915 = rq->i915; 1366 struct intel_engine_cs *engine = rq->engine; 1367 enum intel_engine_id id; 1368 const int num_engines = 1369 IS_HASWELL(i915) ? RUNTIME_INFO(i915)->num_engines - 1 : 0; 1370 bool force_restore = false; 1371 int len; 1372 u32 *cs; 1373 1374 len = 4; 1375 if (IS_GEN(i915, 7)) 1376 len += 2 + (num_engines ? 4 * num_engines + 6 : 0); 1377 else if (IS_GEN(i915, 5)) 1378 len += 2; 1379 if (flags & MI_FORCE_RESTORE) { 1380 GEM_BUG_ON(flags & MI_RESTORE_INHIBIT); 1381 flags &= ~MI_FORCE_RESTORE; 1382 force_restore = true; 1383 len += 2; 1384 } 1385 1386 cs = intel_ring_begin(rq, len); 1387 if (IS_ERR(cs)) 1388 return PTR_ERR(cs); 1389 1390 /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */ 1391 if (IS_GEN(i915, 7)) { 1392 *cs++ = MI_ARB_ON_OFF | MI_ARB_DISABLE; 1393 if (num_engines) { 1394 struct intel_engine_cs *signaller; 1395 1396 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 1397 for_each_engine(signaller, engine->gt, id) { 1398 if (signaller == engine) 1399 continue; 1400 1401 *cs++ = i915_mmio_reg_offset( 1402 RING_PSMI_CTL(signaller->mmio_base)); 1403 *cs++ = _MASKED_BIT_ENABLE( 1404 GEN6_PSMI_SLEEP_MSG_DISABLE); 1405 } 1406 } 1407 } else if (IS_GEN(i915, 5)) { 1408 /* 1409 * This w/a is only listed for pre-production ilk a/b steppings, 1410 * but is also mentioned for programming the powerctx. To be 1411 * safe, just apply the workaround; we do not use SyncFlush so 1412 * this should never take effect and so be a no-op! 1413 */ 1414 *cs++ = MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN; 1415 } 1416 1417 if (force_restore) { 1418 /* 1419 * The HW doesn't handle being told to restore the current 1420 * context very well. Quite often it likes goes to go off and 1421 * sulk, especially when it is meant to be reloading PP_DIR. 1422 * A very simple fix to force the reload is to simply switch 1423 * away from the current context and back again. 1424 * 1425 * Note that the kernel_context will contain random state 1426 * following the INHIBIT_RESTORE. We accept this since we 1427 * never use the kernel_context state; it is merely a 1428 * placeholder we use to flush other contexts. 1429 */ 1430 *cs++ = MI_SET_CONTEXT; 1431 *cs++ = i915_ggtt_offset(engine->kernel_context->state) | 1432 MI_MM_SPACE_GTT | 1433 MI_RESTORE_INHIBIT; 1434 } 1435 1436 *cs++ = MI_NOOP; 1437 *cs++ = MI_SET_CONTEXT; 1438 *cs++ = i915_ggtt_offset(rq->context->state) | flags; 1439 /* 1440 * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP 1441 * WaMiSetContext_Hang:snb,ivb,vlv 1442 */ 1443 *cs++ = MI_NOOP; 1444 1445 if (IS_GEN(i915, 7)) { 1446 if (num_engines) { 1447 struct intel_engine_cs *signaller; 1448 i915_reg_t last_reg = {}; /* keep gcc quiet */ 1449 1450 *cs++ = MI_LOAD_REGISTER_IMM(num_engines); 1451 for_each_engine(signaller, engine->gt, id) { 1452 if (signaller == engine) 1453 continue; 1454 1455 last_reg = RING_PSMI_CTL(signaller->mmio_base); 1456 *cs++ = i915_mmio_reg_offset(last_reg); 1457 *cs++ = _MASKED_BIT_DISABLE( 1458 GEN6_PSMI_SLEEP_MSG_DISABLE); 1459 } 1460 1461 /* Insert a delay before the next switch! */ 1462 *cs++ = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT; 1463 *cs++ = i915_mmio_reg_offset(last_reg); 1464 *cs++ = intel_gt_scratch_offset(engine->gt, 1465 INTEL_GT_SCRATCH_FIELD_DEFAULT); 1466 *cs++ = MI_NOOP; 1467 } 1468 *cs++ = MI_ARB_ON_OFF | MI_ARB_ENABLE; 1469 } else if (IS_GEN(i915, 5)) { 1470 *cs++ = MI_SUSPEND_FLUSH; 1471 } 1472 1473 intel_ring_advance(rq, cs); 1474 1475 return 0; 1476 } 1477 1478 static int remap_l3_slice(struct i915_request *rq, int slice) 1479 { 1480 u32 *cs, *remap_info = rq->i915->l3_parity.remap_info[slice]; 1481 int i; 1482 1483 if (!remap_info) 1484 return 0; 1485 1486 cs = intel_ring_begin(rq, GEN7_L3LOG_SIZE/4 * 2 + 2); 1487 if (IS_ERR(cs)) 1488 return PTR_ERR(cs); 1489 1490 /* 1491 * Note: We do not worry about the concurrent register cacheline hang 1492 * here because no other code should access these registers other than 1493 * at initialization time. 1494 */ 1495 *cs++ = MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4); 1496 for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) { 1497 *cs++ = i915_mmio_reg_offset(GEN7_L3LOG(slice, i)); 1498 *cs++ = remap_info[i]; 1499 } 1500 *cs++ = MI_NOOP; 1501 intel_ring_advance(rq, cs); 1502 1503 return 0; 1504 } 1505 1506 static int remap_l3(struct i915_request *rq) 1507 { 1508 struct i915_gem_context *ctx = i915_request_gem_context(rq); 1509 int i, err; 1510 1511 if (!ctx || !ctx->remap_slice) 1512 return 0; 1513 1514 for (i = 0; i < MAX_L3_SLICES; i++) { 1515 if (!(ctx->remap_slice & BIT(i))) 1516 continue; 1517 1518 err = remap_l3_slice(rq, i); 1519 if (err) 1520 return err; 1521 } 1522 1523 ctx->remap_slice = 0; 1524 return 0; 1525 } 1526 1527 static int switch_mm(struct i915_request *rq, struct i915_address_space *vm) 1528 { 1529 int ret; 1530 1531 if (!vm) 1532 return 0; 1533 1534 ret = rq->engine->emit_flush(rq, EMIT_FLUSH); 1535 if (ret) 1536 return ret; 1537 1538 /* 1539 * Not only do we need a full barrier (post-sync write) after 1540 * invalidating the TLBs, but we need to wait a little bit 1541 * longer. Whether this is merely delaying us, or the 1542 * subsequent flush is a key part of serialising with the 1543 * post-sync op, this extra pass appears vital before a 1544 * mm switch! 1545 */ 1546 ret = load_pd_dir(rq, i915_vm_to_ppgtt(vm), PP_DIR_DCLV_2G); 1547 if (ret) 1548 return ret; 1549 1550 return rq->engine->emit_flush(rq, EMIT_INVALIDATE); 1551 } 1552 1553 static int switch_context(struct i915_request *rq) 1554 { 1555 struct intel_context *ce = rq->context; 1556 int ret; 1557 1558 GEM_BUG_ON(HAS_EXECLISTS(rq->i915)); 1559 1560 ret = switch_mm(rq, vm_alias(ce->vm)); 1561 if (ret) 1562 return ret; 1563 1564 if (ce->state) { 1565 u32 flags; 1566 1567 GEM_BUG_ON(rq->engine->id != RCS0); 1568 1569 /* For resource streamer on HSW+ and power context elsewhere */ 1570 BUILD_BUG_ON(HSW_MI_RS_SAVE_STATE_EN != MI_SAVE_EXT_STATE_EN); 1571 BUILD_BUG_ON(HSW_MI_RS_RESTORE_STATE_EN != MI_RESTORE_EXT_STATE_EN); 1572 1573 flags = MI_SAVE_EXT_STATE_EN | MI_MM_SPACE_GTT; 1574 if (test_bit(CONTEXT_VALID_BIT, &ce->flags)) 1575 flags |= MI_RESTORE_EXT_STATE_EN; 1576 else 1577 flags |= MI_RESTORE_INHIBIT; 1578 1579 ret = mi_set_context(rq, flags); 1580 if (ret) 1581 return ret; 1582 } 1583 1584 ret = remap_l3(rq); 1585 if (ret) 1586 return ret; 1587 1588 return 0; 1589 } 1590 1591 static int ring_request_alloc(struct i915_request *request) 1592 { 1593 int ret; 1594 1595 GEM_BUG_ON(!intel_context_is_pinned(request->context)); 1596 GEM_BUG_ON(i915_request_timeline(request)->has_initial_breadcrumb); 1597 1598 /* 1599 * Flush enough space to reduce the likelihood of waiting after 1600 * we start building the request - in which case we will just 1601 * have to repeat work. 1602 */ 1603 request->reserved_space += LEGACY_REQUEST_SIZE; 1604 1605 /* Unconditionally invalidate GPU caches and TLBs. */ 1606 ret = request->engine->emit_flush(request, EMIT_INVALIDATE); 1607 if (ret) 1608 return ret; 1609 1610 ret = switch_context(request); 1611 if (ret) 1612 return ret; 1613 1614 request->reserved_space -= LEGACY_REQUEST_SIZE; 1615 return 0; 1616 } 1617 1618 static void gen6_bsd_submit_request(struct i915_request *request) 1619 { 1620 struct intel_uncore *uncore = request->engine->uncore; 1621 1622 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1623 1624 /* Every tail move must follow the sequence below */ 1625 1626 /* Disable notification that the ring is IDLE. The GT 1627 * will then assume that it is busy and bring it out of rc6. 1628 */ 1629 intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL, 1630 _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 1631 1632 /* Clear the context id. Here be magic! */ 1633 intel_uncore_write64_fw(uncore, GEN6_BSD_RNCID, 0x0); 1634 1635 /* Wait for the ring not to be idle, i.e. for it to wake up. */ 1636 if (__intel_wait_for_register_fw(uncore, 1637 GEN6_BSD_SLEEP_PSMI_CONTROL, 1638 GEN6_BSD_SLEEP_INDICATOR, 1639 0, 1640 1000, 0, NULL)) 1641 drm_err(&uncore->i915->drm, 1642 "timed out waiting for the BSD ring to wake up\n"); 1643 1644 /* Now that the ring is fully powered up, update the tail */ 1645 i9xx_submit_request(request); 1646 1647 /* Let the ring send IDLE messages to the GT again, 1648 * and so let it sleep to conserve power when idle. 1649 */ 1650 intel_uncore_write_fw(uncore, GEN6_BSD_SLEEP_PSMI_CONTROL, 1651 _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE)); 1652 1653 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1654 } 1655 1656 static int mi_flush_dw(struct i915_request *rq, u32 flags) 1657 { 1658 u32 cmd, *cs; 1659 1660 cs = intel_ring_begin(rq, 4); 1661 if (IS_ERR(cs)) 1662 return PTR_ERR(cs); 1663 1664 cmd = MI_FLUSH_DW; 1665 1666 /* 1667 * We always require a command barrier so that subsequent 1668 * commands, such as breadcrumb interrupts, are strictly ordered 1669 * wrt the contents of the write cache being flushed to memory 1670 * (and thus being coherent from the CPU). 1671 */ 1672 cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW; 1673 1674 /* 1675 * Bspec vol 1c.3 - blitter engine command streamer: 1676 * "If ENABLED, all TLBs will be invalidated once the flush 1677 * operation is complete. This bit is only valid when the 1678 * Post-Sync Operation field is a value of 1h or 3h." 1679 */ 1680 cmd |= flags; 1681 1682 *cs++ = cmd; 1683 *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; 1684 *cs++ = 0; 1685 *cs++ = MI_NOOP; 1686 1687 intel_ring_advance(rq, cs); 1688 1689 return 0; 1690 } 1691 1692 static int gen6_flush_dw(struct i915_request *rq, u32 mode, u32 invflags) 1693 { 1694 return mi_flush_dw(rq, mode & EMIT_INVALIDATE ? invflags : 0); 1695 } 1696 1697 static int gen6_bsd_ring_flush(struct i915_request *rq, u32 mode) 1698 { 1699 return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB | MI_INVALIDATE_BSD); 1700 } 1701 1702 static int 1703 hsw_emit_bb_start(struct i915_request *rq, 1704 u64 offset, u32 len, 1705 unsigned int dispatch_flags) 1706 { 1707 u32 *cs; 1708 1709 cs = intel_ring_begin(rq, 2); 1710 if (IS_ERR(cs)) 1711 return PTR_ERR(cs); 1712 1713 *cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ? 1714 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW); 1715 /* bit0-7 is the length on GEN6+ */ 1716 *cs++ = offset; 1717 intel_ring_advance(rq, cs); 1718 1719 return 0; 1720 } 1721 1722 static int 1723 gen6_emit_bb_start(struct i915_request *rq, 1724 u64 offset, u32 len, 1725 unsigned int dispatch_flags) 1726 { 1727 u32 *cs; 1728 1729 cs = intel_ring_begin(rq, 2); 1730 if (IS_ERR(cs)) 1731 return PTR_ERR(cs); 1732 1733 *cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ? 1734 0 : MI_BATCH_NON_SECURE_I965); 1735 /* bit0-7 is the length on GEN6+ */ 1736 *cs++ = offset; 1737 intel_ring_advance(rq, cs); 1738 1739 return 0; 1740 } 1741 1742 /* Blitter support (SandyBridge+) */ 1743 1744 static int gen6_ring_flush(struct i915_request *rq, u32 mode) 1745 { 1746 return gen6_flush_dw(rq, mode, MI_INVALIDATE_TLB); 1747 } 1748 1749 static void i9xx_set_default_submission(struct intel_engine_cs *engine) 1750 { 1751 engine->submit_request = i9xx_submit_request; 1752 1753 engine->park = NULL; 1754 engine->unpark = NULL; 1755 } 1756 1757 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine) 1758 { 1759 i9xx_set_default_submission(engine); 1760 engine->submit_request = gen6_bsd_submit_request; 1761 } 1762 1763 static void ring_release(struct intel_engine_cs *engine) 1764 { 1765 struct drm_i915_private *dev_priv = engine->i915; 1766 1767 drm_WARN_ON(&dev_priv->drm, INTEL_GEN(dev_priv) > 2 && 1768 (ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE) == 0); 1769 1770 intel_engine_cleanup_common(engine); 1771 1772 intel_ring_unpin(engine->legacy.ring); 1773 intel_ring_put(engine->legacy.ring); 1774 1775 intel_timeline_unpin(engine->legacy.timeline); 1776 intel_timeline_put(engine->legacy.timeline); 1777 } 1778 1779 static void setup_irq(struct intel_engine_cs *engine) 1780 { 1781 struct drm_i915_private *i915 = engine->i915; 1782 1783 if (INTEL_GEN(i915) >= 6) { 1784 engine->irq_enable = gen6_irq_enable; 1785 engine->irq_disable = gen6_irq_disable; 1786 } else if (INTEL_GEN(i915) >= 5) { 1787 engine->irq_enable = gen5_irq_enable; 1788 engine->irq_disable = gen5_irq_disable; 1789 } else if (INTEL_GEN(i915) >= 3) { 1790 engine->irq_enable = i9xx_irq_enable; 1791 engine->irq_disable = i9xx_irq_disable; 1792 } else { 1793 engine->irq_enable = i8xx_irq_enable; 1794 engine->irq_disable = i8xx_irq_disable; 1795 } 1796 } 1797 1798 static void setup_common(struct intel_engine_cs *engine) 1799 { 1800 struct drm_i915_private *i915 = engine->i915; 1801 1802 /* gen8+ are only supported with execlists */ 1803 GEM_BUG_ON(INTEL_GEN(i915) >= 8); 1804 1805 setup_irq(engine); 1806 1807 engine->resume = xcs_resume; 1808 engine->reset.prepare = reset_prepare; 1809 engine->reset.rewind = reset_rewind; 1810 engine->reset.cancel = reset_cancel; 1811 engine->reset.finish = reset_finish; 1812 1813 engine->cops = &ring_context_ops; 1814 engine->request_alloc = ring_request_alloc; 1815 1816 /* 1817 * Using a global execution timeline; the previous final breadcrumb is 1818 * equivalent to our next initial bread so we can elide 1819 * engine->emit_init_breadcrumb(). 1820 */ 1821 engine->emit_fini_breadcrumb = i9xx_emit_breadcrumb; 1822 if (IS_GEN(i915, 5)) 1823 engine->emit_fini_breadcrumb = gen5_emit_breadcrumb; 1824 1825 engine->set_default_submission = i9xx_set_default_submission; 1826 1827 if (INTEL_GEN(i915) >= 6) 1828 engine->emit_bb_start = gen6_emit_bb_start; 1829 else if (INTEL_GEN(i915) >= 4) 1830 engine->emit_bb_start = i965_emit_bb_start; 1831 else if (IS_I830(i915) || IS_I845G(i915)) 1832 engine->emit_bb_start = i830_emit_bb_start; 1833 else 1834 engine->emit_bb_start = i915_emit_bb_start; 1835 } 1836 1837 static void setup_rcs(struct intel_engine_cs *engine) 1838 { 1839 struct drm_i915_private *i915 = engine->i915; 1840 1841 if (HAS_L3_DPF(i915)) 1842 engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT; 1843 1844 engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT; 1845 1846 if (INTEL_GEN(i915) >= 7) { 1847 engine->emit_flush = gen7_render_ring_flush; 1848 engine->emit_fini_breadcrumb = gen7_rcs_emit_breadcrumb; 1849 } else if (IS_GEN(i915, 6)) { 1850 engine->emit_flush = gen6_render_ring_flush; 1851 engine->emit_fini_breadcrumb = gen6_rcs_emit_breadcrumb; 1852 } else if (IS_GEN(i915, 5)) { 1853 engine->emit_flush = gen4_render_ring_flush; 1854 } else { 1855 if (INTEL_GEN(i915) < 4) 1856 engine->emit_flush = gen2_render_ring_flush; 1857 else 1858 engine->emit_flush = gen4_render_ring_flush; 1859 engine->irq_enable_mask = I915_USER_INTERRUPT; 1860 } 1861 1862 if (IS_HASWELL(i915)) 1863 engine->emit_bb_start = hsw_emit_bb_start; 1864 1865 engine->resume = rcs_resume; 1866 } 1867 1868 static void setup_vcs(struct intel_engine_cs *engine) 1869 { 1870 struct drm_i915_private *i915 = engine->i915; 1871 1872 if (INTEL_GEN(i915) >= 6) { 1873 /* gen6 bsd needs a special wa for tail updates */ 1874 if (IS_GEN(i915, 6)) 1875 engine->set_default_submission = gen6_bsd_set_default_submission; 1876 engine->emit_flush = gen6_bsd_ring_flush; 1877 engine->irq_enable_mask = GT_BSD_USER_INTERRUPT; 1878 1879 if (IS_GEN(i915, 6)) 1880 engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb; 1881 else 1882 engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; 1883 } else { 1884 engine->emit_flush = bsd_ring_flush; 1885 if (IS_GEN(i915, 5)) 1886 engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT; 1887 else 1888 engine->irq_enable_mask = I915_BSD_USER_INTERRUPT; 1889 } 1890 } 1891 1892 static void setup_bcs(struct intel_engine_cs *engine) 1893 { 1894 struct drm_i915_private *i915 = engine->i915; 1895 1896 engine->emit_flush = gen6_ring_flush; 1897 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT; 1898 1899 if (IS_GEN(i915, 6)) 1900 engine->emit_fini_breadcrumb = gen6_xcs_emit_breadcrumb; 1901 else 1902 engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; 1903 } 1904 1905 static void setup_vecs(struct intel_engine_cs *engine) 1906 { 1907 struct drm_i915_private *i915 = engine->i915; 1908 1909 GEM_BUG_ON(INTEL_GEN(i915) < 7); 1910 1911 engine->emit_flush = gen6_ring_flush; 1912 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT; 1913 engine->irq_enable = hsw_vebox_irq_enable; 1914 engine->irq_disable = hsw_vebox_irq_disable; 1915 1916 engine->emit_fini_breadcrumb = gen7_xcs_emit_breadcrumb; 1917 } 1918 1919 int intel_ring_submission_setup(struct intel_engine_cs *engine) 1920 { 1921 struct intel_timeline *timeline; 1922 struct intel_ring *ring; 1923 int err; 1924 1925 setup_common(engine); 1926 1927 switch (engine->class) { 1928 case RENDER_CLASS: 1929 setup_rcs(engine); 1930 break; 1931 case VIDEO_DECODE_CLASS: 1932 setup_vcs(engine); 1933 break; 1934 case COPY_ENGINE_CLASS: 1935 setup_bcs(engine); 1936 break; 1937 case VIDEO_ENHANCEMENT_CLASS: 1938 setup_vecs(engine); 1939 break; 1940 default: 1941 MISSING_CASE(engine->class); 1942 return -ENODEV; 1943 } 1944 1945 timeline = intel_timeline_create(engine->gt, engine->status_page.vma); 1946 if (IS_ERR(timeline)) { 1947 err = PTR_ERR(timeline); 1948 goto err; 1949 } 1950 GEM_BUG_ON(timeline->has_initial_breadcrumb); 1951 1952 err = intel_timeline_pin(timeline); 1953 if (err) 1954 goto err_timeline; 1955 1956 ring = intel_engine_create_ring(engine, SZ_16K); 1957 if (IS_ERR(ring)) { 1958 err = PTR_ERR(ring); 1959 goto err_timeline_unpin; 1960 } 1961 1962 err = intel_ring_pin(ring); 1963 if (err) 1964 goto err_ring; 1965 1966 GEM_BUG_ON(engine->legacy.ring); 1967 engine->legacy.ring = ring; 1968 engine->legacy.timeline = timeline; 1969 1970 GEM_BUG_ON(timeline->hwsp_ggtt != engine->status_page.vma); 1971 1972 /* Finally, take ownership and responsibility for cleanup! */ 1973 engine->release = ring_release; 1974 1975 return 0; 1976 1977 err_ring: 1978 intel_ring_put(ring); 1979 err_timeline_unpin: 1980 intel_timeline_unpin(timeline); 1981 err_timeline: 1982 intel_timeline_put(timeline); 1983 err: 1984 intel_engine_cleanup_common(engine); 1985 return err; 1986 } 1987