1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include "gem/i915_gem_context.h" 8 #include "gem/i915_gem_pm.h" 9 10 #include "i915_drv.h" 11 #include "i915_globals.h" 12 13 #include "intel_context.h" 14 #include "intel_engine.h" 15 #include "intel_engine_pm.h" 16 #include "intel_ring.h" 17 18 static struct i915_global_context { 19 struct i915_global base; 20 struct kmem_cache *slab_ce; 21 } global; 22 23 static struct intel_context *intel_context_alloc(void) 24 { 25 return kmem_cache_zalloc(global.slab_ce, GFP_KERNEL); 26 } 27 28 void intel_context_free(struct intel_context *ce) 29 { 30 kmem_cache_free(global.slab_ce, ce); 31 } 32 33 struct intel_context * 34 intel_context_create(struct intel_engine_cs *engine) 35 { 36 struct intel_context *ce; 37 38 ce = intel_context_alloc(); 39 if (!ce) 40 return ERR_PTR(-ENOMEM); 41 42 intel_context_init(ce, engine); 43 return ce; 44 } 45 46 int intel_context_alloc_state(struct intel_context *ce) 47 { 48 int err = 0; 49 50 if (mutex_lock_interruptible(&ce->pin_mutex)) 51 return -EINTR; 52 53 if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { 54 err = ce->ops->alloc(ce); 55 if (unlikely(err)) 56 goto unlock; 57 58 set_bit(CONTEXT_ALLOC_BIT, &ce->flags); 59 } 60 61 unlock: 62 mutex_unlock(&ce->pin_mutex); 63 return err; 64 } 65 66 static int intel_context_active_acquire(struct intel_context *ce) 67 { 68 int err; 69 70 __i915_active_acquire(&ce->active); 71 72 if (intel_context_is_barrier(ce)) 73 return 0; 74 75 /* Preallocate tracking nodes */ 76 err = i915_active_acquire_preallocate_barrier(&ce->active, 77 ce->engine); 78 if (err) 79 i915_active_release(&ce->active); 80 81 return err; 82 } 83 84 static void intel_context_active_release(struct intel_context *ce) 85 { 86 /* Nodes preallocated in intel_context_active() */ 87 i915_active_acquire_barrier(&ce->active); 88 i915_active_release(&ce->active); 89 } 90 91 int __intel_context_do_pin(struct intel_context *ce) 92 { 93 int err; 94 95 if (unlikely(!test_bit(CONTEXT_ALLOC_BIT, &ce->flags))) { 96 err = intel_context_alloc_state(ce); 97 if (err) 98 return err; 99 } 100 101 err = i915_active_acquire(&ce->active); 102 if (err) 103 return err; 104 105 if (mutex_lock_interruptible(&ce->pin_mutex)) { 106 err = -EINTR; 107 goto out_release; 108 } 109 110 if (likely(!atomic_add_unless(&ce->pin_count, 1, 0))) { 111 err = intel_context_active_acquire(ce); 112 if (unlikely(err)) 113 goto out_unlock; 114 115 err = ce->ops->pin(ce); 116 if (unlikely(err)) 117 goto err_active; 118 119 CE_TRACE(ce, "pin ring:{start:%08x, head:%04x, tail:%04x}\n", 120 i915_ggtt_offset(ce->ring->vma), 121 ce->ring->head, ce->ring->tail); 122 123 smp_mb__before_atomic(); /* flush pin before it is visible */ 124 atomic_inc(&ce->pin_count); 125 } 126 127 GEM_BUG_ON(!intel_context_is_pinned(ce)); /* no overflow! */ 128 GEM_BUG_ON(i915_active_is_idle(&ce->active)); 129 goto out_unlock; 130 131 err_active: 132 intel_context_active_release(ce); 133 out_unlock: 134 mutex_unlock(&ce->pin_mutex); 135 out_release: 136 i915_active_release(&ce->active); 137 return err; 138 } 139 140 void intel_context_unpin(struct intel_context *ce) 141 { 142 if (!atomic_dec_and_test(&ce->pin_count)) 143 return; 144 145 CE_TRACE(ce, "unpin\n"); 146 ce->ops->unpin(ce); 147 148 /* 149 * Once released, we may asynchronously drop the active reference. 150 * As that may be the only reference keeping the context alive, 151 * take an extra now so that it is not freed before we finish 152 * dereferencing it. 153 */ 154 intel_context_get(ce); 155 intel_context_active_release(ce); 156 intel_context_put(ce); 157 } 158 159 static int __context_pin_state(struct i915_vma *vma) 160 { 161 unsigned int bias = i915_ggtt_pin_bias(vma) | PIN_OFFSET_BIAS; 162 int err; 163 164 err = i915_ggtt_pin(vma, 0, bias | PIN_HIGH); 165 if (err) 166 return err; 167 168 err = i915_active_acquire(&vma->active); 169 if (err) 170 goto err_unpin; 171 172 /* 173 * And mark it as a globally pinned object to let the shrinker know 174 * it cannot reclaim the object until we release it. 175 */ 176 i915_vma_make_unshrinkable(vma); 177 vma->obj->mm.dirty = true; 178 179 return 0; 180 181 err_unpin: 182 i915_vma_unpin(vma); 183 return err; 184 } 185 186 static void __context_unpin_state(struct i915_vma *vma) 187 { 188 i915_vma_make_shrinkable(vma); 189 i915_active_release(&vma->active); 190 __i915_vma_unpin(vma); 191 } 192 193 static int __ring_active(struct intel_ring *ring) 194 { 195 int err; 196 197 err = i915_active_acquire(&ring->vma->active); 198 if (err) 199 return err; 200 201 err = intel_ring_pin(ring); 202 if (err) 203 goto err_active; 204 205 return 0; 206 207 err_active: 208 i915_active_release(&ring->vma->active); 209 return err; 210 } 211 212 static void __ring_retire(struct intel_ring *ring) 213 { 214 intel_ring_unpin(ring); 215 i915_active_release(&ring->vma->active); 216 } 217 218 __i915_active_call 219 static void __intel_context_retire(struct i915_active *active) 220 { 221 struct intel_context *ce = container_of(active, typeof(*ce), active); 222 223 CE_TRACE(ce, "retire runtime: { total:%lluns, avg:%lluns }\n", 224 intel_context_get_total_runtime_ns(ce), 225 intel_context_get_avg_runtime_ns(ce)); 226 227 set_bit(CONTEXT_VALID_BIT, &ce->flags); 228 if (ce->state) 229 __context_unpin_state(ce->state); 230 231 intel_timeline_unpin(ce->timeline); 232 __ring_retire(ce->ring); 233 234 intel_context_put(ce); 235 } 236 237 static int __intel_context_active(struct i915_active *active) 238 { 239 struct intel_context *ce = container_of(active, typeof(*ce), active); 240 int err; 241 242 CE_TRACE(ce, "active\n"); 243 244 intel_context_get(ce); 245 246 err = __ring_active(ce->ring); 247 if (err) 248 goto err_put; 249 250 err = intel_timeline_pin(ce->timeline); 251 if (err) 252 goto err_ring; 253 254 if (!ce->state) 255 return 0; 256 257 err = __context_pin_state(ce->state); 258 if (err) 259 goto err_timeline; 260 261 return 0; 262 263 err_timeline: 264 intel_timeline_unpin(ce->timeline); 265 err_ring: 266 __ring_retire(ce->ring); 267 err_put: 268 intel_context_put(ce); 269 return err; 270 } 271 272 void 273 intel_context_init(struct intel_context *ce, 274 struct intel_engine_cs *engine) 275 { 276 GEM_BUG_ON(!engine->cops); 277 GEM_BUG_ON(!engine->gt->vm); 278 279 kref_init(&ce->ref); 280 281 ce->engine = engine; 282 ce->ops = engine->cops; 283 ce->sseu = engine->sseu; 284 ce->ring = __intel_context_ring_size(SZ_4K); 285 286 ewma_runtime_init(&ce->runtime.avg); 287 288 ce->vm = i915_vm_get(engine->gt->vm); 289 290 INIT_LIST_HEAD(&ce->signal_link); 291 INIT_LIST_HEAD(&ce->signals); 292 293 mutex_init(&ce->pin_mutex); 294 295 i915_active_init(&ce->active, 296 __intel_context_active, __intel_context_retire); 297 } 298 299 void intel_context_fini(struct intel_context *ce) 300 { 301 if (ce->timeline) 302 intel_timeline_put(ce->timeline); 303 i915_vm_put(ce->vm); 304 305 mutex_destroy(&ce->pin_mutex); 306 i915_active_fini(&ce->active); 307 } 308 309 static void i915_global_context_shrink(void) 310 { 311 kmem_cache_shrink(global.slab_ce); 312 } 313 314 static void i915_global_context_exit(void) 315 { 316 kmem_cache_destroy(global.slab_ce); 317 } 318 319 static struct i915_global_context global = { { 320 .shrink = i915_global_context_shrink, 321 .exit = i915_global_context_exit, 322 } }; 323 324 int __init i915_global_context_init(void) 325 { 326 global.slab_ce = KMEM_CACHE(intel_context, SLAB_HWCACHE_ALIGN); 327 if (!global.slab_ce) 328 return -ENOMEM; 329 330 i915_global_register(&global.base); 331 return 0; 332 } 333 334 void intel_context_enter_engine(struct intel_context *ce) 335 { 336 intel_engine_pm_get(ce->engine); 337 intel_timeline_enter(ce->timeline); 338 } 339 340 void intel_context_exit_engine(struct intel_context *ce) 341 { 342 intel_timeline_exit(ce->timeline); 343 intel_engine_pm_put(ce->engine); 344 } 345 346 int intel_context_prepare_remote_request(struct intel_context *ce, 347 struct i915_request *rq) 348 { 349 struct intel_timeline *tl = ce->timeline; 350 int err; 351 352 /* Only suitable for use in remotely modifying this context */ 353 GEM_BUG_ON(rq->context == ce); 354 355 if (rcu_access_pointer(rq->timeline) != tl) { /* timeline sharing! */ 356 /* Queue this switch after current activity by this context. */ 357 err = i915_active_fence_set(&tl->last_request, rq); 358 if (err) 359 return err; 360 } 361 362 /* 363 * Guarantee context image and the timeline remains pinned until the 364 * modifying request is retired by setting the ce activity tracker. 365 * 366 * But we only need to take one pin on the account of it. Or in other 367 * words transfer the pinned ce object to tracked active request. 368 */ 369 GEM_BUG_ON(i915_active_is_idle(&ce->active)); 370 return i915_active_add_request(&ce->active, rq); 371 } 372 373 struct i915_request *intel_context_create_request(struct intel_context *ce) 374 { 375 struct i915_request *rq; 376 int err; 377 378 err = intel_context_pin(ce); 379 if (unlikely(err)) 380 return ERR_PTR(err); 381 382 rq = i915_request_create(ce); 383 intel_context_unpin(ce); 384 385 return rq; 386 } 387 388 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 389 #include "selftest_context.c" 390 #endif 391