1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2024 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include <linux/sched.h> 25 #include <drm/drm_exec.h> 26 #include "amdgpu.h" 27 28 #define work_to_evf_mgr(w, name) container_of(w, struct amdgpu_eviction_fence_mgr, name) 29 #define evf_mgr_to_fpriv(e) container_of(e, struct amdgpu_fpriv, evf_mgr) 30 31 static const char * 32 amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence) 33 { 34 return "amdgpu_eviction_fence"; 35 } 36 37 static const char * 38 amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f) 39 { 40 struct amdgpu_eviction_fence *ef; 41 42 ef = container_of(f, struct amdgpu_eviction_fence, base); 43 return ef->timeline_name; 44 } 45 46 int 47 amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, 48 struct drm_exec *exec) 49 { 50 struct amdgpu_eviction_fence *old_ef, *new_ef; 51 struct drm_gem_object *obj; 52 unsigned long index; 53 int ret; 54 55 if (evf_mgr->ev_fence && 56 !dma_fence_is_signaled(&evf_mgr->ev_fence->base)) 57 return 0; 58 /* 59 * Steps to replace eviction fence: 60 * * lock all objects in exec (caller) 61 * * create a new eviction fence 62 * * update new eviction fence in evf_mgr 63 * * attach the new eviction fence to BOs 64 * * release the old fence 65 * * unlock the objects (caller) 66 */ 67 new_ef = amdgpu_eviction_fence_create(evf_mgr); 68 if (!new_ef) { 69 DRM_ERROR("Failed to create new eviction fence\n"); 70 return -ENOMEM; 71 } 72 73 /* Update the eviction fence now */ 74 spin_lock(&evf_mgr->ev_fence_lock); 75 old_ef = evf_mgr->ev_fence; 76 evf_mgr->ev_fence = new_ef; 77 spin_unlock(&evf_mgr->ev_fence_lock); 78 79 /* Attach the new fence */ 80 drm_exec_for_each_locked_object(exec, index, obj) { 81 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 82 83 if (!bo) 84 continue; 85 ret = amdgpu_eviction_fence_attach(evf_mgr, bo); 86 if (ret) { 87 DRM_ERROR("Failed to attch new eviction fence\n"); 88 goto free_err; 89 } 90 } 91 92 /* Free old fence */ 93 if (old_ef) 94 dma_fence_put(&old_ef->base); 95 return 0; 96 97 free_err: 98 kfree(new_ef); 99 return ret; 100 } 101 102 static void 103 amdgpu_eviction_fence_suspend_worker(struct work_struct *work) 104 { 105 struct amdgpu_eviction_fence_mgr *evf_mgr = work_to_evf_mgr(work, suspend_work.work); 106 struct amdgpu_fpriv *fpriv = evf_mgr_to_fpriv(evf_mgr); 107 struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; 108 struct amdgpu_eviction_fence *ev_fence; 109 110 mutex_lock(&uq_mgr->userq_mutex); 111 spin_lock(&evf_mgr->ev_fence_lock); 112 ev_fence = evf_mgr->ev_fence; 113 if (ev_fence) 114 dma_fence_get(&ev_fence->base); 115 else 116 goto unlock; 117 spin_unlock(&evf_mgr->ev_fence_lock); 118 119 amdgpu_userq_evict(uq_mgr, ev_fence); 120 121 mutex_unlock(&uq_mgr->userq_mutex); 122 dma_fence_put(&ev_fence->base); 123 return; 124 125 unlock: 126 spin_unlock(&evf_mgr->ev_fence_lock); 127 mutex_unlock(&uq_mgr->userq_mutex); 128 } 129 130 static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f) 131 { 132 struct amdgpu_eviction_fence_mgr *evf_mgr; 133 struct amdgpu_eviction_fence *ev_fence; 134 135 if (!f) 136 return true; 137 138 ev_fence = to_ev_fence(f); 139 evf_mgr = ev_fence->evf_mgr; 140 141 schedule_delayed_work(&evf_mgr->suspend_work, 0); 142 return true; 143 } 144 145 static const struct dma_fence_ops amdgpu_eviction_fence_ops = { 146 .get_driver_name = amdgpu_eviction_fence_get_driver_name, 147 .get_timeline_name = amdgpu_eviction_fence_get_timeline_name, 148 .enable_signaling = amdgpu_eviction_fence_enable_signaling, 149 }; 150 151 void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr, 152 struct amdgpu_eviction_fence *ev_fence) 153 { 154 spin_lock(&evf_mgr->ev_fence_lock); 155 dma_fence_signal(&ev_fence->base); 156 spin_unlock(&evf_mgr->ev_fence_lock); 157 } 158 159 struct amdgpu_eviction_fence * 160 amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr) 161 { 162 struct amdgpu_eviction_fence *ev_fence; 163 164 ev_fence = kzalloc(sizeof(*ev_fence), GFP_KERNEL); 165 if (!ev_fence) 166 return NULL; 167 168 ev_fence->evf_mgr = evf_mgr; 169 get_task_comm(ev_fence->timeline_name, current); 170 spin_lock_init(&ev_fence->lock); 171 dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops, 172 &ev_fence->lock, evf_mgr->ev_fence_ctx, 173 atomic_inc_return(&evf_mgr->ev_fence_seq)); 174 return ev_fence; 175 } 176 177 void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr) 178 { 179 struct amdgpu_eviction_fence *ev_fence; 180 181 /* Wait for any pending work to execute */ 182 flush_delayed_work(&evf_mgr->suspend_work); 183 184 spin_lock(&evf_mgr->ev_fence_lock); 185 ev_fence = evf_mgr->ev_fence; 186 spin_unlock(&evf_mgr->ev_fence_lock); 187 188 if (!ev_fence) 189 return; 190 191 dma_fence_wait(&ev_fence->base, false); 192 193 /* Last unref of ev_fence */ 194 dma_fence_put(&ev_fence->base); 195 } 196 197 int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr, 198 struct amdgpu_bo *bo) 199 { 200 struct amdgpu_eviction_fence *ev_fence; 201 struct dma_resv *resv = bo->tbo.base.resv; 202 int ret; 203 204 if (!resv) 205 return 0; 206 207 ret = dma_resv_reserve_fences(resv, 1); 208 if (ret) { 209 DRM_DEBUG_DRIVER("Failed to resv fence space\n"); 210 return ret; 211 } 212 213 spin_lock(&evf_mgr->ev_fence_lock); 214 ev_fence = evf_mgr->ev_fence; 215 if (ev_fence) 216 dma_resv_add_fence(resv, &ev_fence->base, DMA_RESV_USAGE_BOOKKEEP); 217 spin_unlock(&evf_mgr->ev_fence_lock); 218 219 return 0; 220 } 221 222 void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr, 223 struct amdgpu_bo *bo) 224 { 225 struct dma_fence *stub = dma_fence_get_stub(); 226 227 dma_resv_replace_fences(bo->tbo.base.resv, evf_mgr->ev_fence_ctx, 228 stub, DMA_RESV_USAGE_BOOKKEEP); 229 dma_fence_put(stub); 230 } 231 232 int amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr) 233 { 234 /* This needs to be done one time per open */ 235 atomic_set(&evf_mgr->ev_fence_seq, 0); 236 evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1); 237 spin_lock_init(&evf_mgr->ev_fence_lock); 238 239 INIT_DELAYED_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker); 240 return 0; 241 } 242