1 /* 2 * Copyright 2016-2018 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 */ 22 23 #include <linux/dma-fence.h> 24 #include <linux/spinlock.h> 25 #include <linux/atomic.h> 26 #include <linux/stacktrace.h> 27 #include <linux/sched.h> 28 #include <linux/slab.h> 29 #include <linux/sched/mm.h> 30 #include "amdgpu_amdkfd.h" 31 #include "kfd_svm.h" 32 33 static const struct dma_fence_ops amdkfd_fence_ops; 34 static atomic_t fence_seq = ATOMIC_INIT(0); 35 36 /* Eviction Fence 37 * Fence helper functions to deal with KFD memory eviction. 38 * Big Idea - Since KFD submissions are done by user queues, a BO cannot be 39 * evicted unless all the user queues for that process are evicted. 40 * 41 * All the BOs in a process share an eviction fence. When process X wants 42 * to map VRAM memory but TTM can't find enough space, TTM will attempt to 43 * evict BOs from its LRU list. TTM checks if the BO is valuable to evict 44 * by calling ttm_device_funcs->eviction_valuable(). 45 * 46 * ttm_device_funcs->eviction_valuable() - will return false if the BO belongs 47 * to process X. Otherwise, it will return true to indicate BO can be 48 * evicted by TTM. 49 * 50 * If ttm_device_funcs->eviction_valuable returns true, then TTM will continue 51 * the evcition process for that BO by calling ttm_bo_evict --> amdgpu_bo_move 52 * --> amdgpu_copy_buffer(). This sets up job in GPU scheduler. 53 * 54 * GPU Scheduler (amd_sched_main) - sets up a cb (fence_add_callback) to 55 * nofity when the BO is free to move. fence_add_callback --> enable_signaling 56 * --> amdgpu_amdkfd_fence.enable_signaling 57 * 58 * amdgpu_amdkfd_fence.enable_signaling - Start a work item that will quiesce 59 * user queues and signal fence. The work item will also start another delayed 60 * work item to restore BOs 61 */ 62 63 struct amdgpu_amdkfd_fence *amdgpu_amdkfd_fence_create(u64 context, 64 struct mm_struct *mm, 65 struct svm_range_bo *svm_bo, 66 u16 context_id) 67 { 68 struct amdgpu_amdkfd_fence *fence; 69 70 fence = kzalloc(sizeof(*fence), GFP_KERNEL); 71 if (fence == NULL) 72 return NULL; 73 74 /* This reference gets released in amdkfd_fence_release */ 75 mmgrab(mm); 76 fence->mm = mm; 77 get_task_comm(fence->timeline_name, current); 78 spin_lock_init(&fence->lock); 79 fence->svm_bo = svm_bo; 80 fence->context_id = context_id; 81 dma_fence_init(&fence->base, &amdkfd_fence_ops, &fence->lock, 82 context, atomic_inc_return(&fence_seq)); 83 84 return fence; 85 } 86 87 struct amdgpu_amdkfd_fence *to_amdgpu_amdkfd_fence(struct dma_fence *f) 88 { 89 struct amdgpu_amdkfd_fence *fence; 90 91 if (!f) 92 return NULL; 93 94 fence = container_of(f, struct amdgpu_amdkfd_fence, base); 95 if (f->ops == &amdkfd_fence_ops) 96 return fence; 97 98 return NULL; 99 } 100 101 static const char *amdkfd_fence_get_driver_name(struct dma_fence *f) 102 { 103 return "amdgpu_amdkfd_fence"; 104 } 105 106 static const char *amdkfd_fence_get_timeline_name(struct dma_fence *f) 107 { 108 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); 109 110 return fence->timeline_name; 111 } 112 113 /** 114 * amdkfd_fence_enable_signaling - This gets called when TTM wants to evict 115 * a KFD BO and schedules a job to move the BO. 116 * If fence is already signaled return true. 117 * If fence is not signaled schedule a evict KFD process work item. 118 * 119 * @f: dma_fence 120 */ 121 static bool amdkfd_fence_enable_signaling(struct dma_fence *f) 122 { 123 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); 124 125 if (!fence) 126 return false; 127 128 if (dma_fence_is_signaled(f)) 129 return true; 130 131 /* if fence->svm_bo is NULL, means this fence is created through 132 * init_kfd_vm() or amdgpu_amdkfd_gpuvm_restore_process_bos(). 133 * Therefore, this fence is amdgpu_amdkfd_fence->eviction_fence. 134 */ 135 if (!fence->svm_bo) { 136 if (!kgd2kfd_schedule_evict_and_restore_process(fence->mm, fence->context_id, f)) 137 return true; 138 } else { 139 if (!svm_range_schedule_evict_svm_bo(fence)) 140 return true; 141 } 142 return false; 143 } 144 145 /** 146 * amdkfd_fence_release - callback that fence can be freed 147 * 148 * @f: dma_fence 149 * 150 * This function is called when the reference count becomes zero. 151 * Drops the mm_struct reference and RCU schedules freeing up the fence. 152 */ 153 static void amdkfd_fence_release(struct dma_fence *f) 154 { 155 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); 156 157 /* Unconditionally signal the fence. The process is getting 158 * terminated. 159 */ 160 if (WARN_ON(!fence)) 161 return; /* Not an amdgpu_amdkfd_fence */ 162 163 mmdrop(fence->mm); 164 kfree_rcu(f, rcu); 165 } 166 167 /** 168 * amdkfd_fence_check_mm - Check whether to prevent eviction of @f by @mm 169 * 170 * @f: [IN] fence 171 * @mm: [IN] mm that needs to be verified 172 * 173 * Check if @mm is same as that of the fence @f, if same return TRUE else 174 * return FALSE. 175 * For svm bo, which support vram overcommitment, always return FALSE. 176 */ 177 bool amdkfd_fence_check_mm(struct dma_fence *f, struct mm_struct *mm) 178 { 179 struct amdgpu_amdkfd_fence *fence = to_amdgpu_amdkfd_fence(f); 180 181 if (!fence) 182 return false; 183 else if (fence->mm == mm && !fence->svm_bo) 184 return true; 185 186 return false; 187 } 188 189 static const struct dma_fence_ops amdkfd_fence_ops = { 190 .get_driver_name = amdkfd_fence_get_driver_name, 191 .get_timeline_name = amdkfd_fence_get_timeline_name, 192 .enable_signaling = amdkfd_fence_enable_signaling, 193 .release = amdkfd_fence_release, 194 }; 195