1 /* 2 * Copyright 2014 Advanced Micro Devices, Inc. 3 * All Rights Reserved. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the 7 * "Software"), to deal in the Software without restriction, including 8 * without limitation the rights to use, copy, modify, merge, publish, 9 * distribute, sub license, and/or sell copies of the Software, and to 10 * permit persons to whom the Software is furnished to do so, subject to 11 * the following conditions: 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 19 * USE OR OTHER DEALINGS IN THE SOFTWARE. 20 * 21 * The above copyright notice and this permission notice (including the 22 * next paragraph) shall be included in all copies or substantial portions 23 * of the Software. 24 * 25 */ 26 /* 27 * Authors: 28 * Christian König <christian.koenig@amd.com> 29 */ 30 31 #include <linux/dma-fence-chain.h> 32 33 #include "amdgpu.h" 34 #include "amdgpu_trace.h" 35 #include "amdgpu_amdkfd.h" 36 37 struct amdgpu_sync_entry { 38 struct hlist_node node; 39 struct dma_fence *fence; 40 }; 41 42 static struct kmem_cache *amdgpu_sync_slab; 43 44 /** 45 * amdgpu_sync_create - zero init sync object 46 * 47 * @sync: sync object to initialize 48 * 49 * Just clear the sync object for now. 50 */ 51 void amdgpu_sync_create(struct amdgpu_sync *sync) 52 { 53 hash_init(sync->fences); 54 } 55 56 /** 57 * amdgpu_sync_same_dev - test if fence belong to us 58 * 59 * @adev: amdgpu device to use for the test 60 * @f: fence to test 61 * 62 * Test if the fence was issued by us. 63 */ 64 static bool amdgpu_sync_same_dev(struct amdgpu_device *adev, 65 struct dma_fence *f) 66 { 67 struct drm_sched_fence *s_fence = to_drm_sched_fence(f); 68 69 if (s_fence) { 70 struct amdgpu_ring *ring; 71 72 ring = container_of(s_fence->sched, struct amdgpu_ring, sched); 73 return ring->adev == adev; 74 } 75 76 return false; 77 } 78 79 /** 80 * amdgpu_sync_get_owner - extract the owner of a fence 81 * 82 * @f: fence get the owner from 83 * 84 * Extract who originally created the fence. 85 */ 86 static void *amdgpu_sync_get_owner(struct dma_fence *f) 87 { 88 struct drm_sched_fence *s_fence; 89 struct amdgpu_amdkfd_fence *kfd_fence; 90 91 if (!f) 92 return AMDGPU_FENCE_OWNER_UNDEFINED; 93 94 s_fence = to_drm_sched_fence(f); 95 if (s_fence) 96 return s_fence->owner; 97 98 kfd_fence = to_amdgpu_amdkfd_fence(f); 99 if (kfd_fence) 100 return AMDGPU_FENCE_OWNER_KFD; 101 102 return AMDGPU_FENCE_OWNER_UNDEFINED; 103 } 104 105 /** 106 * amdgpu_sync_keep_later - Keep the later fence 107 * 108 * @keep: existing fence to test 109 * @fence: new fence 110 * 111 * Either keep the existing fence or the new one, depending which one is later. 112 */ 113 static void amdgpu_sync_keep_later(struct dma_fence **keep, 114 struct dma_fence *fence) 115 { 116 if (*keep && dma_fence_is_later(*keep, fence)) 117 return; 118 119 dma_fence_put(*keep); 120 *keep = dma_fence_get(fence); 121 } 122 123 /** 124 * amdgpu_sync_add_later - add the fence to the hash 125 * 126 * @sync: sync object to add the fence to 127 * @f: fence to add 128 * 129 * Tries to add the fence to an existing hash entry. Returns true when an entry 130 * was found, false otherwise. 131 */ 132 static bool amdgpu_sync_add_later(struct amdgpu_sync *sync, struct dma_fence *f) 133 { 134 struct amdgpu_sync_entry *e; 135 136 hash_for_each_possible(sync->fences, e, node, f->context) { 137 if (unlikely(e->fence->context != f->context)) 138 continue; 139 140 amdgpu_sync_keep_later(&e->fence, f); 141 return true; 142 } 143 return false; 144 } 145 146 /** 147 * amdgpu_sync_fence - remember to sync to this fence 148 * 149 * @sync: sync object to add fence to 150 * @f: fence to sync to 151 * 152 * Add the fence to the sync object. 153 */ 154 int amdgpu_sync_fence(struct amdgpu_sync *sync, struct dma_fence *f) 155 { 156 struct amdgpu_sync_entry *e; 157 158 if (!f) 159 return 0; 160 161 if (amdgpu_sync_add_later(sync, f)) 162 return 0; 163 164 e = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL); 165 if (!e) 166 return -ENOMEM; 167 168 hash_add(sync->fences, &e->node, f->context); 169 e->fence = dma_fence_get(f); 170 return 0; 171 } 172 173 /* Determine based on the owner and mode if we should sync to a fence or not */ 174 static bool amdgpu_sync_test_fence(struct amdgpu_device *adev, 175 enum amdgpu_sync_mode mode, 176 void *owner, struct dma_fence *f) 177 { 178 void *fence_owner = amdgpu_sync_get_owner(f); 179 180 /* Always sync to moves, no matter what */ 181 if (fence_owner == AMDGPU_FENCE_OWNER_UNDEFINED) 182 return true; 183 184 /* We only want to trigger KFD eviction fences on 185 * evict or move jobs. Skip KFD fences otherwise. 186 */ 187 if (fence_owner == AMDGPU_FENCE_OWNER_KFD && 188 owner != AMDGPU_FENCE_OWNER_UNDEFINED) 189 return false; 190 191 /* Never sync to VM updates either. */ 192 if (fence_owner == AMDGPU_FENCE_OWNER_VM && 193 owner != AMDGPU_FENCE_OWNER_UNDEFINED) 194 return false; 195 196 /* Ignore fences depending on the sync mode */ 197 switch (mode) { 198 case AMDGPU_SYNC_ALWAYS: 199 return true; 200 201 case AMDGPU_SYNC_NE_OWNER: 202 if (amdgpu_sync_same_dev(adev, f) && 203 fence_owner == owner) 204 return false; 205 break; 206 207 case AMDGPU_SYNC_EQ_OWNER: 208 if (amdgpu_sync_same_dev(adev, f) && 209 fence_owner != owner) 210 return false; 211 break; 212 213 case AMDGPU_SYNC_EXPLICIT: 214 return false; 215 } 216 217 WARN(debug_evictions && fence_owner == AMDGPU_FENCE_OWNER_KFD, 218 "Adding eviction fence to sync obj"); 219 return true; 220 } 221 222 /** 223 * amdgpu_sync_resv - sync to a reservation object 224 * 225 * @adev: amdgpu device 226 * @sync: sync object to add fences from reservation object to 227 * @resv: reservation object with embedded fence 228 * @mode: how owner affects which fences we sync to 229 * @owner: owner of the planned job submission 230 * 231 * Sync to the fence 232 */ 233 int amdgpu_sync_resv(struct amdgpu_device *adev, struct amdgpu_sync *sync, 234 struct dma_resv *resv, enum amdgpu_sync_mode mode, 235 void *owner) 236 { 237 struct dma_resv_iter cursor; 238 struct dma_fence *f; 239 int r; 240 241 if (resv == NULL) 242 return -EINVAL; 243 244 /* TODO: Use DMA_RESV_USAGE_READ here */ 245 dma_resv_for_each_fence(&cursor, resv, DMA_RESV_USAGE_BOOKKEEP, f) { 246 dma_fence_chain_for_each(f, f) { 247 struct dma_fence *tmp = dma_fence_chain_contained(f); 248 249 if (amdgpu_sync_test_fence(adev, mode, owner, tmp)) { 250 r = amdgpu_sync_fence(sync, f); 251 dma_fence_put(f); 252 if (r) 253 return r; 254 break; 255 } 256 } 257 } 258 return 0; 259 } 260 261 /** 262 * amdgpu_sync_peek_fence - get the next fence not signaled yet 263 * 264 * @sync: the sync object 265 * @ring: optional ring to use for test 266 * 267 * Returns the next fence not signaled yet without removing it from the sync 268 * object. 269 */ 270 struct dma_fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync, 271 struct amdgpu_ring *ring) 272 { 273 struct amdgpu_sync_entry *e; 274 struct hlist_node *tmp; 275 int i; 276 277 hash_for_each_safe(sync->fences, i, tmp, e, node) { 278 struct dma_fence *f = e->fence; 279 struct drm_sched_fence *s_fence = to_drm_sched_fence(f); 280 281 if (dma_fence_is_signaled(f)) { 282 hash_del(&e->node); 283 dma_fence_put(f); 284 kmem_cache_free(amdgpu_sync_slab, e); 285 continue; 286 } 287 if (ring && s_fence) { 288 /* For fences from the same ring it is sufficient 289 * when they are scheduled. 290 */ 291 if (s_fence->sched == &ring->sched) { 292 if (dma_fence_is_signaled(&s_fence->scheduled)) 293 continue; 294 295 return &s_fence->scheduled; 296 } 297 } 298 299 return f; 300 } 301 302 return NULL; 303 } 304 305 /** 306 * amdgpu_sync_get_fence - get the next fence from the sync object 307 * 308 * @sync: sync object to use 309 * 310 * Get and removes the next fence from the sync object not signaled yet. 311 */ 312 struct dma_fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync) 313 { 314 struct amdgpu_sync_entry *e; 315 struct hlist_node *tmp; 316 struct dma_fence *f; 317 int i; 318 hash_for_each_safe(sync->fences, i, tmp, e, node) { 319 320 f = e->fence; 321 322 hash_del(&e->node); 323 kmem_cache_free(amdgpu_sync_slab, e); 324 325 if (!dma_fence_is_signaled(f)) 326 return f; 327 328 dma_fence_put(f); 329 } 330 return NULL; 331 } 332 333 /** 334 * amdgpu_sync_clone - clone a sync object 335 * 336 * @source: sync object to clone 337 * @clone: pointer to destination sync object 338 * 339 * Adds references to all unsignaled fences in @source to @clone. Also 340 * removes signaled fences from @source while at it. 341 */ 342 int amdgpu_sync_clone(struct amdgpu_sync *source, struct amdgpu_sync *clone) 343 { 344 struct amdgpu_sync_entry *e; 345 struct hlist_node *tmp; 346 struct dma_fence *f; 347 int i, r; 348 349 hash_for_each_safe(source->fences, i, tmp, e, node) { 350 f = e->fence; 351 if (!dma_fence_is_signaled(f)) { 352 r = amdgpu_sync_fence(clone, f); 353 if (r) 354 return r; 355 } else { 356 hash_del(&e->node); 357 dma_fence_put(f); 358 kmem_cache_free(amdgpu_sync_slab, e); 359 } 360 } 361 362 return 0; 363 } 364 365 int amdgpu_sync_wait(struct amdgpu_sync *sync, bool intr) 366 { 367 struct amdgpu_sync_entry *e; 368 struct hlist_node *tmp; 369 int i, r; 370 371 hash_for_each_safe(sync->fences, i, tmp, e, node) { 372 r = dma_fence_wait(e->fence, intr); 373 if (r) 374 return r; 375 376 hash_del(&e->node); 377 dma_fence_put(e->fence); 378 kmem_cache_free(amdgpu_sync_slab, e); 379 } 380 381 return 0; 382 } 383 384 /** 385 * amdgpu_sync_free - free the sync object 386 * 387 * @sync: sync object to use 388 * 389 * Free the sync object. 390 */ 391 void amdgpu_sync_free(struct amdgpu_sync *sync) 392 { 393 struct amdgpu_sync_entry *e; 394 struct hlist_node *tmp; 395 unsigned i; 396 397 hash_for_each_safe(sync->fences, i, tmp, e, node) { 398 hash_del(&e->node); 399 dma_fence_put(e->fence); 400 kmem_cache_free(amdgpu_sync_slab, e); 401 } 402 } 403 404 /** 405 * amdgpu_sync_init - init sync object subsystem 406 * 407 * Allocate the slab allocator. 408 */ 409 int amdgpu_sync_init(void) 410 { 411 amdgpu_sync_slab = kmem_cache_create( 412 "amdgpu_sync", sizeof(struct amdgpu_sync_entry), 0, 413 SLAB_HWCACHE_ALIGN, NULL); 414 if (!amdgpu_sync_slab) 415 return -ENOMEM; 416 417 return 0; 418 } 419 420 /** 421 * amdgpu_sync_fini - fini sync object subsystem 422 * 423 * Free the slab allocator. 424 */ 425 void amdgpu_sync_fini(void) 426 { 427 kmem_cache_destroy(amdgpu_sync_slab); 428 } 429