1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2018 Intel Corporation 5 */ 6 #include "gt/intel_gpu_commands.h" 7 #include "gt/intel_gt.h" 8 9 #include "gem/i915_gem_internal.h" 10 #include "gem/selftests/igt_gem_utils.h" 11 12 #include "igt_spinner.h" 13 14 int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt) 15 { 16 int err; 17 18 memset(spin, 0, sizeof(*spin)); 19 spin->gt = gt; 20 21 spin->hws = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 22 if (IS_ERR(spin->hws)) { 23 err = PTR_ERR(spin->hws); 24 goto err; 25 } 26 i915_gem_object_set_cache_coherency(spin->hws, I915_CACHE_LLC); 27 28 spin->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE); 29 if (IS_ERR(spin->obj)) { 30 err = PTR_ERR(spin->obj); 31 goto err_hws; 32 } 33 34 return 0; 35 36 err_hws: 37 i915_gem_object_put(spin->hws); 38 err: 39 return err; 40 } 41 42 static void *igt_spinner_pin_obj(struct intel_context *ce, 43 struct i915_gem_ww_ctx *ww, 44 struct drm_i915_gem_object *obj, 45 unsigned int mode, struct i915_vma **vma) 46 { 47 void *vaddr; 48 int ret; 49 50 *vma = i915_vma_instance(obj, ce->vm, NULL); 51 if (IS_ERR(*vma)) 52 return ERR_CAST(*vma); 53 54 ret = i915_gem_object_lock(obj, ww); 55 if (ret) 56 return ERR_PTR(ret); 57 58 vaddr = i915_gem_object_pin_map(obj, mode); 59 60 if (!ww) 61 i915_gem_object_unlock(obj); 62 63 if (IS_ERR(vaddr)) 64 return vaddr; 65 66 if (ww) 67 ret = i915_vma_pin_ww(*vma, ww, 0, 0, PIN_USER); 68 else 69 ret = i915_vma_pin(*vma, 0, 0, PIN_USER); 70 71 if (ret) { 72 i915_gem_object_unpin_map(obj); 73 return ERR_PTR(ret); 74 } 75 76 return vaddr; 77 } 78 79 int igt_spinner_pin(struct igt_spinner *spin, 80 struct intel_context *ce, 81 struct i915_gem_ww_ctx *ww) 82 { 83 void *vaddr; 84 85 if (spin->ce && WARN_ON(spin->ce != ce)) 86 return -ENODEV; 87 spin->ce = ce; 88 89 if (!spin->seqno) { 90 vaddr = igt_spinner_pin_obj(ce, ww, spin->hws, I915_MAP_WB, &spin->hws_vma); 91 if (IS_ERR(vaddr)) 92 return PTR_ERR(vaddr); 93 94 spin->seqno = memset(vaddr, 0xff, PAGE_SIZE); 95 } 96 97 if (!spin->batch) { 98 unsigned int mode; 99 100 mode = i915_coherent_map_type(spin->gt->i915, spin->obj, false); 101 vaddr = igt_spinner_pin_obj(ce, ww, spin->obj, mode, &spin->batch_vma); 102 if (IS_ERR(vaddr)) 103 return PTR_ERR(vaddr); 104 105 spin->batch = vaddr; 106 } 107 108 return 0; 109 } 110 111 static unsigned int seqno_offset(u64 fence) 112 { 113 return offset_in_page(sizeof(u32) * fence); 114 } 115 116 static u64 hws_address(const struct i915_vma *hws, 117 const struct i915_request *rq) 118 { 119 return hws->node.start + seqno_offset(rq->fence.context); 120 } 121 122 static int move_to_active(struct i915_vma *vma, 123 struct i915_request *rq, 124 unsigned int flags) 125 { 126 int err; 127 128 i915_vma_lock(vma); 129 err = i915_request_await_object(rq, vma->obj, 130 flags & EXEC_OBJECT_WRITE); 131 if (err == 0) 132 err = i915_vma_move_to_active(vma, rq, flags); 133 i915_vma_unlock(vma); 134 135 return err; 136 } 137 138 struct i915_request * 139 igt_spinner_create_request(struct igt_spinner *spin, 140 struct intel_context *ce, 141 u32 arbitration_command) 142 { 143 struct intel_engine_cs *engine = ce->engine; 144 struct i915_request *rq = NULL; 145 struct i915_vma *hws, *vma; 146 unsigned int flags; 147 u32 *batch; 148 int err; 149 150 GEM_BUG_ON(spin->gt != ce->vm->gt); 151 152 if (!intel_engine_can_store_dword(ce->engine)) 153 return ERR_PTR(-ENODEV); 154 155 if (!spin->batch) { 156 err = igt_spinner_pin(spin, ce, NULL); 157 if (err) 158 return ERR_PTR(err); 159 } 160 161 hws = spin->hws_vma; 162 vma = spin->batch_vma; 163 164 rq = intel_context_create_request(ce); 165 if (IS_ERR(rq)) 166 return ERR_CAST(rq); 167 168 err = move_to_active(vma, rq, 0); 169 if (err) 170 goto cancel_rq; 171 172 err = move_to_active(hws, rq, 0); 173 if (err) 174 goto cancel_rq; 175 176 batch = spin->batch; 177 178 if (GRAPHICS_VER(rq->engine->i915) >= 8) { 179 *batch++ = MI_STORE_DWORD_IMM_GEN4; 180 *batch++ = lower_32_bits(hws_address(hws, rq)); 181 *batch++ = upper_32_bits(hws_address(hws, rq)); 182 } else if (GRAPHICS_VER(rq->engine->i915) >= 6) { 183 *batch++ = MI_STORE_DWORD_IMM_GEN4; 184 *batch++ = 0; 185 *batch++ = hws_address(hws, rq); 186 } else if (GRAPHICS_VER(rq->engine->i915) >= 4) { 187 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 188 *batch++ = 0; 189 *batch++ = hws_address(hws, rq); 190 } else { 191 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 192 *batch++ = hws_address(hws, rq); 193 } 194 *batch++ = rq->fence.seqno; 195 196 *batch++ = arbitration_command; 197 198 if (GRAPHICS_VER(rq->engine->i915) >= 8) 199 *batch++ = MI_BATCH_BUFFER_START | BIT(8) | 1; 200 else if (IS_HASWELL(rq->engine->i915)) 201 *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW; 202 else if (GRAPHICS_VER(rq->engine->i915) >= 6) 203 *batch++ = MI_BATCH_BUFFER_START; 204 else 205 *batch++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 206 *batch++ = lower_32_bits(vma->node.start); 207 *batch++ = upper_32_bits(vma->node.start); 208 209 *batch++ = MI_BATCH_BUFFER_END; /* not reached */ 210 211 intel_gt_chipset_flush(engine->gt); 212 213 if (engine->emit_init_breadcrumb) { 214 err = engine->emit_init_breadcrumb(rq); 215 if (err) 216 goto cancel_rq; 217 } 218 219 flags = 0; 220 if (GRAPHICS_VER(rq->engine->i915) <= 5) 221 flags |= I915_DISPATCH_SECURE; 222 err = engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags); 223 224 cancel_rq: 225 if (err) { 226 i915_request_set_error_once(rq, err); 227 i915_request_add(rq); 228 } 229 return err ? ERR_PTR(err) : rq; 230 } 231 232 static u32 233 hws_seqno(const struct igt_spinner *spin, const struct i915_request *rq) 234 { 235 u32 *seqno = spin->seqno + seqno_offset(rq->fence.context); 236 237 return READ_ONCE(*seqno); 238 } 239 240 void igt_spinner_end(struct igt_spinner *spin) 241 { 242 if (!spin->batch) 243 return; 244 245 *spin->batch = MI_BATCH_BUFFER_END; 246 intel_gt_chipset_flush(spin->gt); 247 } 248 249 void igt_spinner_fini(struct igt_spinner *spin) 250 { 251 igt_spinner_end(spin); 252 253 if (spin->batch) { 254 i915_vma_unpin(spin->batch_vma); 255 i915_gem_object_unpin_map(spin->obj); 256 } 257 i915_gem_object_put(spin->obj); 258 259 if (spin->seqno) { 260 i915_vma_unpin(spin->hws_vma); 261 i915_gem_object_unpin_map(spin->hws); 262 } 263 i915_gem_object_put(spin->hws); 264 } 265 266 bool igt_wait_for_spinner(struct igt_spinner *spin, struct i915_request *rq) 267 { 268 if (i915_request_is_ready(rq)) 269 intel_engine_flush_submission(rq->engine); 270 271 return !(wait_for_us(i915_seqno_passed(hws_seqno(spin, rq), 272 rq->fence.seqno), 273 100) && 274 wait_for(i915_seqno_passed(hws_seqno(spin, rq), 275 rq->fence.seqno), 276 50)); 277 } 278