1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2024 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include <linux/sched.h> 25 #include <drm/drm_exec.h> 26 #include "amdgpu.h" 27 28 #define work_to_evf_mgr(w, name) container_of(w, struct amdgpu_eviction_fence_mgr, name) 29 #define evf_mgr_to_fpriv(e) container_of(e, struct amdgpu_fpriv, evf_mgr) 30 31 static const char * 32 amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence) 33 { 34 return "amdgpu_eviction_fence"; 35 } 36 37 static const char * 38 amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f) 39 { 40 struct amdgpu_eviction_fence *ef; 41 42 ef = container_of(f, struct amdgpu_eviction_fence, base); 43 return ef->timeline_name; 44 } 45 46 int 47 amdgpu_eviction_fence_replace_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, 48 struct drm_exec *exec) 49 { 50 struct amdgpu_eviction_fence *old_ef, *new_ef; 51 struct drm_gem_object *obj; 52 unsigned long index; 53 int ret; 54 55 if (evf_mgr->ev_fence && 56 !dma_fence_is_signaled(&evf_mgr->ev_fence->base)) 57 return 0; 58 /* 59 * Steps to replace eviction fence: 60 * * lock all objects in exec (caller) 61 * * create a new eviction fence 62 * * update new eviction fence in evf_mgr 63 * * attach the new eviction fence to BOs 64 * * release the old fence 65 * * unlock the objects (caller) 66 */ 67 new_ef = amdgpu_eviction_fence_create(evf_mgr); 68 if (!new_ef) { 69 DRM_ERROR("Failed to create new eviction fence\n"); 70 return -ENOMEM; 71 } 72 73 /* Update the eviction fence now */ 74 spin_lock(&evf_mgr->ev_fence_lock); 75 old_ef = evf_mgr->ev_fence; 76 evf_mgr->ev_fence = new_ef; 77 spin_unlock(&evf_mgr->ev_fence_lock); 78 79 /* Attach the new fence */ 80 drm_exec_for_each_locked_object(exec, index, obj) { 81 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 82 83 if (!bo) 84 continue; 85 ret = amdgpu_eviction_fence_attach(evf_mgr, bo); 86 if (ret) { 87 DRM_ERROR("Failed to attch new eviction fence\n"); 88 goto free_err; 89 } 90 } 91 92 /* Free old fence */ 93 if (old_ef) 94 dma_fence_put(&old_ef->base); 95 return 0; 96 97 free_err: 98 kfree(new_ef); 99 return ret; 100 } 101 102 static void 103 amdgpu_eviction_fence_suspend_worker(struct work_struct *work) 104 { 105 struct amdgpu_eviction_fence_mgr *evf_mgr = work_to_evf_mgr(work, suspend_work.work); 106 struct amdgpu_fpriv *fpriv = evf_mgr_to_fpriv(evf_mgr); 107 struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; 108 struct amdgpu_eviction_fence *ev_fence; 109 110 mutex_lock(&uq_mgr->userq_mutex); 111 spin_lock(&evf_mgr->ev_fence_lock); 112 ev_fence = evf_mgr->ev_fence; 113 if (ev_fence) 114 dma_fence_get(&ev_fence->base); 115 else 116 goto unlock; 117 spin_unlock(&evf_mgr->ev_fence_lock); 118 119 amdgpu_userq_evict(uq_mgr, ev_fence); 120 121 mutex_unlock(&uq_mgr->userq_mutex); 122 dma_fence_put(&ev_fence->base); 123 return; 124 125 unlock: 126 spin_unlock(&evf_mgr->ev_fence_lock); 127 mutex_unlock(&uq_mgr->userq_mutex); 128 } 129 130 static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f) 131 { 132 struct amdgpu_eviction_fence_mgr *evf_mgr; 133 struct amdgpu_eviction_fence *ev_fence; 134 135 if (!f) 136 return true; 137 138 ev_fence = to_ev_fence(f); 139 evf_mgr = ev_fence->evf_mgr; 140 141 schedule_delayed_work(&evf_mgr->suspend_work, 0); 142 return true; 143 } 144 145 static const struct dma_fence_ops amdgpu_eviction_fence_ops = { 146 .use_64bit_seqno = true, 147 .get_driver_name = amdgpu_eviction_fence_get_driver_name, 148 .get_timeline_name = amdgpu_eviction_fence_get_timeline_name, 149 .enable_signaling = amdgpu_eviction_fence_enable_signaling, 150 }; 151 152 void amdgpu_eviction_fence_signal(struct amdgpu_eviction_fence_mgr *evf_mgr, 153 struct amdgpu_eviction_fence *ev_fence) 154 { 155 spin_lock(&evf_mgr->ev_fence_lock); 156 dma_fence_signal(&ev_fence->base); 157 spin_unlock(&evf_mgr->ev_fence_lock); 158 } 159 160 struct amdgpu_eviction_fence * 161 amdgpu_eviction_fence_create(struct amdgpu_eviction_fence_mgr *evf_mgr) 162 { 163 struct amdgpu_eviction_fence *ev_fence; 164 165 ev_fence = kzalloc(sizeof(*ev_fence), GFP_KERNEL); 166 if (!ev_fence) 167 return NULL; 168 169 ev_fence->evf_mgr = evf_mgr; 170 get_task_comm(ev_fence->timeline_name, current); 171 spin_lock_init(&ev_fence->lock); 172 dma_fence_init(&ev_fence->base, &amdgpu_eviction_fence_ops, 173 &ev_fence->lock, evf_mgr->ev_fence_ctx, 174 atomic_inc_return(&evf_mgr->ev_fence_seq)); 175 return ev_fence; 176 } 177 178 void amdgpu_eviction_fence_destroy(struct amdgpu_eviction_fence_mgr *evf_mgr) 179 { 180 struct amdgpu_eviction_fence *ev_fence; 181 182 /* Wait for any pending work to execute */ 183 flush_delayed_work(&evf_mgr->suspend_work); 184 185 spin_lock(&evf_mgr->ev_fence_lock); 186 ev_fence = evf_mgr->ev_fence; 187 spin_unlock(&evf_mgr->ev_fence_lock); 188 189 if (!ev_fence) 190 return; 191 192 dma_fence_wait(&ev_fence->base, false); 193 194 /* Last unref of ev_fence */ 195 dma_fence_put(&ev_fence->base); 196 } 197 198 int amdgpu_eviction_fence_attach(struct amdgpu_eviction_fence_mgr *evf_mgr, 199 struct amdgpu_bo *bo) 200 { 201 struct amdgpu_eviction_fence *ev_fence; 202 struct dma_resv *resv = bo->tbo.base.resv; 203 int ret; 204 205 if (!resv) 206 return 0; 207 208 ret = dma_resv_reserve_fences(resv, 1); 209 if (ret) { 210 DRM_DEBUG_DRIVER("Failed to resv fence space\n"); 211 return ret; 212 } 213 214 spin_lock(&evf_mgr->ev_fence_lock); 215 ev_fence = evf_mgr->ev_fence; 216 if (ev_fence) 217 dma_resv_add_fence(resv, &ev_fence->base, DMA_RESV_USAGE_BOOKKEEP); 218 spin_unlock(&evf_mgr->ev_fence_lock); 219 220 return 0; 221 } 222 223 void amdgpu_eviction_fence_detach(struct amdgpu_eviction_fence_mgr *evf_mgr, 224 struct amdgpu_bo *bo) 225 { 226 struct dma_fence *stub = dma_fence_get_stub(); 227 228 dma_resv_replace_fences(bo->tbo.base.resv, evf_mgr->ev_fence_ctx, 229 stub, DMA_RESV_USAGE_BOOKKEEP); 230 dma_fence_put(stub); 231 } 232 233 int amdgpu_eviction_fence_init(struct amdgpu_eviction_fence_mgr *evf_mgr) 234 { 235 /* This needs to be done one time per open */ 236 atomic_set(&evf_mgr->ev_fence_seq, 0); 237 evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1); 238 spin_lock_init(&evf_mgr->ev_fence_lock); 239 240 INIT_DELAYED_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker); 241 return 0; 242 } 243