1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2019 Intel Corporation 4 */ 5 6 #include "gen7_renderclear.h" 7 #include "i915_drv.h" 8 #include "intel_gpu_commands.h" 9 10 #define GT3_INLINE_DATA_DELAYS 0x1E00 11 #define batch_advance(Y, CS) GEM_BUG_ON((Y)->end != (CS)) 12 13 struct cb_kernel { 14 const void *data; 15 u32 size; 16 }; 17 18 #define CB_KERNEL(name) { .data = (name), .size = sizeof(name) } 19 20 #include "ivb_clear_kernel.c" 21 static const struct cb_kernel cb_kernel_ivb = CB_KERNEL(ivb_clear_kernel); 22 23 #include "hsw_clear_kernel.c" 24 static const struct cb_kernel cb_kernel_hsw = CB_KERNEL(hsw_clear_kernel); 25 26 struct batch_chunk { 27 struct i915_vma *vma; 28 u32 offset; 29 u32 *start; 30 u32 *end; 31 u32 max_items; 32 }; 33 34 struct batch_vals { 35 u32 max_threads; 36 u32 state_start; 37 u32 surface_start; 38 u32 surface_height; 39 u32 surface_width; 40 u32 size; 41 }; 42 43 static int num_primitives(const struct batch_vals *bv) 44 { 45 /* 46 * We need to saturate the GPU with work in order to dispatch 47 * a shader on every HW thread, and clear the thread-local registers. 48 * In short, we have to dispatch work faster than the shaders can 49 * run in order to fill the EU and occupy each HW thread. 50 */ 51 return bv->max_threads; 52 } 53 54 static void 55 batch_get_defaults(struct drm_i915_private *i915, struct batch_vals *bv) 56 { 57 if (IS_HASWELL(i915)) { 58 switch (INTEL_INFO(i915)->gt) { 59 default: 60 case 1: 61 bv->max_threads = 70; 62 break; 63 case 2: 64 bv->max_threads = 140; 65 break; 66 case 3: 67 bv->max_threads = 280; 68 break; 69 } 70 bv->surface_height = 16 * 16; 71 bv->surface_width = 32 * 2 * 16; 72 } else { 73 switch (INTEL_INFO(i915)->gt) { 74 default: 75 case 1: /* including vlv */ 76 bv->max_threads = 36; 77 break; 78 case 2: 79 bv->max_threads = 128; 80 break; 81 } 82 bv->surface_height = 16 * 8; 83 bv->surface_width = 32 * 16; 84 } 85 bv->state_start = round_up(SZ_1K + num_primitives(bv) * 64, SZ_4K); 86 bv->surface_start = bv->state_start + SZ_4K; 87 bv->size = bv->surface_start + bv->surface_height * bv->surface_width; 88 } 89 90 static void batch_init(struct batch_chunk *bc, 91 struct i915_vma *vma, 92 u32 *start, u32 offset, u32 max_bytes) 93 { 94 bc->vma = vma; 95 bc->offset = offset; 96 bc->start = start + bc->offset / sizeof(*bc->start); 97 bc->end = bc->start; 98 bc->max_items = max_bytes / sizeof(*bc->start); 99 } 100 101 static u32 batch_offset(const struct batch_chunk *bc, u32 *cs) 102 { 103 return (cs - bc->start) * sizeof(*bc->start) + bc->offset; 104 } 105 106 static u32 batch_addr(const struct batch_chunk *bc) 107 { 108 return bc->vma->node.start; 109 } 110 111 static void batch_add(struct batch_chunk *bc, const u32 d) 112 { 113 GEM_BUG_ON((bc->end - bc->start) >= bc->max_items); 114 *bc->end++ = d; 115 } 116 117 static u32 *batch_alloc_items(struct batch_chunk *bc, u32 align, u32 items) 118 { 119 u32 *map; 120 121 if (align) { 122 u32 *end = PTR_ALIGN(bc->end, align); 123 124 memset32(bc->end, 0, end - bc->end); 125 bc->end = end; 126 } 127 128 map = bc->end; 129 bc->end += items; 130 131 return map; 132 } 133 134 static u32 *batch_alloc_bytes(struct batch_chunk *bc, u32 align, u32 bytes) 135 { 136 GEM_BUG_ON(!IS_ALIGNED(bytes, sizeof(*bc->start))); 137 return batch_alloc_items(bc, align, bytes / sizeof(*bc->start)); 138 } 139 140 static u32 141 gen7_fill_surface_state(struct batch_chunk *state, 142 const u32 dst_offset, 143 const struct batch_vals *bv) 144 { 145 u32 surface_h = bv->surface_height; 146 u32 surface_w = bv->surface_width; 147 u32 *cs = batch_alloc_items(state, 32, 8); 148 u32 offset = batch_offset(state, cs); 149 150 #define SURFACE_2D 1 151 #define SURFACEFORMAT_B8G8R8A8_UNORM 0x0C0 152 #define RENDER_CACHE_READ_WRITE 1 153 154 *cs++ = SURFACE_2D << 29 | 155 (SURFACEFORMAT_B8G8R8A8_UNORM << 18) | 156 (RENDER_CACHE_READ_WRITE << 8); 157 158 *cs++ = batch_addr(state) + dst_offset; 159 160 *cs++ = ((surface_h / 4 - 1) << 16) | (surface_w / 4 - 1); 161 *cs++ = surface_w; 162 *cs++ = 0; 163 *cs++ = 0; 164 *cs++ = 0; 165 #define SHADER_CHANNELS(r, g, b, a) \ 166 (((r) << 25) | ((g) << 22) | ((b) << 19) | ((a) << 16)) 167 *cs++ = SHADER_CHANNELS(4, 5, 6, 7); 168 batch_advance(state, cs); 169 170 return offset; 171 } 172 173 static u32 174 gen7_fill_binding_table(struct batch_chunk *state, 175 const struct batch_vals *bv) 176 { 177 u32 surface_start = 178 gen7_fill_surface_state(state, bv->surface_start, bv); 179 u32 *cs = batch_alloc_items(state, 32, 8); 180 u32 offset = batch_offset(state, cs); 181 182 *cs++ = surface_start - state->offset; 183 *cs++ = 0; 184 *cs++ = 0; 185 *cs++ = 0; 186 *cs++ = 0; 187 *cs++ = 0; 188 *cs++ = 0; 189 *cs++ = 0; 190 batch_advance(state, cs); 191 192 return offset; 193 } 194 195 static u32 196 gen7_fill_kernel_data(struct batch_chunk *state, 197 const u32 *data, 198 const u32 size) 199 { 200 return batch_offset(state, 201 memcpy(batch_alloc_bytes(state, 64, size), 202 data, size)); 203 } 204 205 static u32 206 gen7_fill_interface_descriptor(struct batch_chunk *state, 207 const struct batch_vals *bv, 208 const struct cb_kernel *kernel, 209 unsigned int count) 210 { 211 u32 kernel_offset = 212 gen7_fill_kernel_data(state, kernel->data, kernel->size); 213 u32 binding_table = gen7_fill_binding_table(state, bv); 214 u32 *cs = batch_alloc_items(state, 32, 8 * count); 215 u32 offset = batch_offset(state, cs); 216 217 *cs++ = kernel_offset; 218 *cs++ = (1 << 7) | (1 << 13); 219 *cs++ = 0; 220 *cs++ = (binding_table - state->offset) | 1; 221 *cs++ = 0; 222 *cs++ = 0; 223 *cs++ = 0; 224 *cs++ = 0; 225 226 /* 1 - 63dummy idds */ 227 memset32(cs, 0x00, (count - 1) * 8); 228 batch_advance(state, cs + (count - 1) * 8); 229 230 return offset; 231 } 232 233 static void 234 gen7_emit_state_base_address(struct batch_chunk *batch, 235 u32 surface_state_base) 236 { 237 u32 *cs = batch_alloc_items(batch, 0, 10); 238 239 *cs++ = STATE_BASE_ADDRESS | (10 - 2); 240 /* general */ 241 *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY; 242 /* surface */ 243 *cs++ = (batch_addr(batch) + surface_state_base) | BASE_ADDRESS_MODIFY; 244 /* dynamic */ 245 *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY; 246 /* indirect */ 247 *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY; 248 /* instruction */ 249 *cs++ = batch_addr(batch) | BASE_ADDRESS_MODIFY; 250 251 /* general/dynamic/indirect/instruction access Bound */ 252 *cs++ = 0; 253 *cs++ = BASE_ADDRESS_MODIFY; 254 *cs++ = 0; 255 *cs++ = BASE_ADDRESS_MODIFY; 256 batch_advance(batch, cs); 257 } 258 259 static void 260 gen7_emit_vfe_state(struct batch_chunk *batch, 261 const struct batch_vals *bv, 262 u32 urb_size, u32 curbe_size, 263 u32 mode) 264 { 265 u32 threads = bv->max_threads - 1; 266 u32 *cs = batch_alloc_items(batch, 32, 8); 267 268 *cs++ = MEDIA_VFE_STATE | (8 - 2); 269 270 /* scratch buffer */ 271 *cs++ = 0; 272 273 /* number of threads & urb entries for GPGPU vs Media Mode */ 274 *cs++ = threads << 16 | 1 << 8 | mode << 2; 275 276 *cs++ = 0; 277 278 /* urb entry size & curbe size in 256 bits unit */ 279 *cs++ = urb_size << 16 | curbe_size; 280 281 /* scoreboard */ 282 *cs++ = 0; 283 *cs++ = 0; 284 *cs++ = 0; 285 batch_advance(batch, cs); 286 } 287 288 static void 289 gen7_emit_interface_descriptor_load(struct batch_chunk *batch, 290 const u32 interface_descriptor, 291 unsigned int count) 292 { 293 u32 *cs = batch_alloc_items(batch, 8, 4); 294 295 *cs++ = MEDIA_INTERFACE_DESCRIPTOR_LOAD | (4 - 2); 296 *cs++ = 0; 297 *cs++ = count * 8 * sizeof(*cs); 298 299 /* 300 * interface descriptor address - it is relative to the dynamics base 301 * address 302 */ 303 *cs++ = interface_descriptor; 304 batch_advance(batch, cs); 305 } 306 307 static void 308 gen7_emit_media_object(struct batch_chunk *batch, 309 unsigned int media_object_index) 310 { 311 unsigned int x_offset = (media_object_index % 16) * 64; 312 unsigned int y_offset = (media_object_index / 16) * 16; 313 unsigned int pkt = 6 + 3; 314 u32 *cs; 315 316 cs = batch_alloc_items(batch, 8, pkt); 317 318 *cs++ = MEDIA_OBJECT | (pkt - 2); 319 320 /* interface descriptor offset */ 321 *cs++ = 0; 322 323 /* without indirect data */ 324 *cs++ = 0; 325 *cs++ = 0; 326 327 /* scoreboard */ 328 *cs++ = 0; 329 *cs++ = 0; 330 331 /* inline */ 332 *cs++ = y_offset << 16 | x_offset; 333 *cs++ = 0; 334 *cs++ = GT3_INLINE_DATA_DELAYS; 335 336 batch_advance(batch, cs); 337 } 338 339 static void gen7_emit_pipeline_flush(struct batch_chunk *batch) 340 { 341 u32 *cs = batch_alloc_items(batch, 0, 4); 342 343 *cs++ = GFX_OP_PIPE_CONTROL(4); 344 *cs++ = PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | 345 PIPE_CONTROL_DEPTH_CACHE_FLUSH | 346 PIPE_CONTROL_DC_FLUSH_ENABLE | 347 PIPE_CONTROL_CS_STALL; 348 *cs++ = 0; 349 *cs++ = 0; 350 351 batch_advance(batch, cs); 352 } 353 354 static void gen7_emit_pipeline_invalidate(struct batch_chunk *batch) 355 { 356 u32 *cs = batch_alloc_items(batch, 0, 10); 357 358 /* ivb: Stall before STATE_CACHE_INVALIDATE */ 359 *cs++ = GFX_OP_PIPE_CONTROL(5); 360 *cs++ = PIPE_CONTROL_STALL_AT_SCOREBOARD | 361 PIPE_CONTROL_CS_STALL; 362 *cs++ = 0; 363 *cs++ = 0; 364 *cs++ = 0; 365 366 *cs++ = GFX_OP_PIPE_CONTROL(5); 367 *cs++ = PIPE_CONTROL_STATE_CACHE_INVALIDATE; 368 *cs++ = 0; 369 *cs++ = 0; 370 *cs++ = 0; 371 372 batch_advance(batch, cs); 373 } 374 375 static void emit_batch(struct i915_vma * const vma, 376 u32 *start, 377 const struct batch_vals *bv) 378 { 379 struct drm_i915_private *i915 = vma->vm->i915; 380 const unsigned int desc_count = 1; 381 const unsigned int urb_size = 1; 382 struct batch_chunk cmds, state; 383 u32 descriptors; 384 unsigned int i; 385 386 batch_init(&cmds, vma, start, 0, bv->state_start); 387 batch_init(&state, vma, start, bv->state_start, SZ_4K); 388 389 descriptors = gen7_fill_interface_descriptor(&state, bv, 390 IS_HASWELL(i915) ? 391 &cb_kernel_hsw : 392 &cb_kernel_ivb, 393 desc_count); 394 395 /* Reset inherited context registers */ 396 gen7_emit_pipeline_flush(&cmds); 397 gen7_emit_pipeline_invalidate(&cmds); 398 batch_add(&cmds, MI_LOAD_REGISTER_IMM(2)); 399 batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_0_GEN7)); 400 batch_add(&cmds, 0xffff0000 | 401 ((IS_IVB_GT1(i915) || IS_VALLEYVIEW(i915)) ? 402 HIZ_RAW_STALL_OPT_DISABLE : 403 0)); 404 batch_add(&cmds, i915_mmio_reg_offset(CACHE_MODE_1)); 405 batch_add(&cmds, 0xffff0000 | PIXEL_SUBSPAN_COLLECT_OPT_DISABLE); 406 gen7_emit_pipeline_invalidate(&cmds); 407 gen7_emit_pipeline_flush(&cmds); 408 409 /* Switch to the media pipeline and our base address */ 410 gen7_emit_pipeline_invalidate(&cmds); 411 batch_add(&cmds, PIPELINE_SELECT | PIPELINE_SELECT_MEDIA); 412 batch_add(&cmds, MI_NOOP); 413 gen7_emit_pipeline_invalidate(&cmds); 414 415 gen7_emit_pipeline_flush(&cmds); 416 gen7_emit_state_base_address(&cmds, descriptors); 417 gen7_emit_pipeline_invalidate(&cmds); 418 419 /* Set the clear-residual kernel state */ 420 gen7_emit_vfe_state(&cmds, bv, urb_size - 1, 0, 0); 421 gen7_emit_interface_descriptor_load(&cmds, descriptors, desc_count); 422 423 /* Execute the kernel on all HW threads */ 424 for (i = 0; i < num_primitives(bv); i++) 425 gen7_emit_media_object(&cmds, i); 426 427 batch_add(&cmds, MI_BATCH_BUFFER_END); 428 } 429 430 int gen7_setup_clear_gpr_bb(struct intel_engine_cs * const engine, 431 struct i915_vma * const vma) 432 { 433 struct batch_vals bv; 434 u32 *batch; 435 436 batch_get_defaults(engine->i915, &bv); 437 if (!vma) 438 return bv.size; 439 440 GEM_BUG_ON(vma->obj->base.size < bv.size); 441 442 batch = i915_gem_object_pin_map(vma->obj, I915_MAP_WC); 443 if (IS_ERR(batch)) 444 return PTR_ERR(batch); 445 446 emit_batch(vma, memset(batch, 0, bv.size), &bv); 447 448 i915_gem_object_flush_map(vma->obj); 449 __i915_gem_object_release_map(vma->obj); 450 451 return 0; 452 } 453