1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2024 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include <linux/sched.h> 25 #include <drm/drm_exec.h> 26 #include "amdgpu.h" 27 28 static const char * 29 amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence) 30 { 31 return "amdgpu_eviction_fence"; 32 } 33 34 static const char * 35 amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f) 36 { 37 struct amdgpu_eviction_fence *ef; 38 39 ef = container_of(f, struct amdgpu_eviction_fence, base); 40 return ef->timeline_name; 41 } 42 43 static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f) 44 { 45 struct amdgpu_eviction_fence *ev_fence = to_ev_fence(f); 46 47 schedule_work(&ev_fence->evf_mgr->suspend_work); 48 return true; 49 } 50 51 static const struct dma_fence_ops amdgpu_eviction_fence_ops = { 52 .get_driver_name = amdgpu_eviction_fence_get_driver_name, 53 .get_timeline_name = amdgpu_eviction_fence_get_timeline_name, 54 .enable_signaling = amdgpu_eviction_fence_enable_signaling, 55 }; 56 57 static void 58 amdgpu_eviction_fence_suspend_worker(struct work_struct *work) 59 { 60 struct amdgpu_eviction_fence_mgr *evf_mgr = 61 container_of(work, struct amdgpu_eviction_fence_mgr, 62 suspend_work); 63 struct amdgpu_fpriv *fpriv = 64 container_of(evf_mgr, struct amdgpu_fpriv, evf_mgr); 65 struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; 66 struct dma_fence *ev_fence; 67 bool cookie; 68 69 mutex_lock(&uq_mgr->userq_mutex); 70 71 /* Fence waits are not allowed in a fence signalling critical section. */ 72 amdgpu_userq_wait_for_signal(uq_mgr); 73 74 /* 75 * This is intentionally after taking the userq_mutex since we do 76 * allocate memory while holding this lock, but only after ensuring that 77 * the eviction fence is signaled. 78 */ 79 cookie = dma_fence_begin_signalling(); 80 81 ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr); 82 amdgpu_userq_evict(uq_mgr); 83 84 /* 85 * Signaling the eviction fence must be done while holding the 86 * userq_mutex. Otherwise we won't resume the queues before issuing the 87 * next fence. 88 */ 89 dma_fence_signal(ev_fence); 90 dma_fence_end_signalling(cookie); 91 dma_fence_put(ev_fence); 92 93 if (!evf_mgr->shutdown) 94 schedule_delayed_work(&uq_mgr->resume_work, 0); 95 96 mutex_unlock(&uq_mgr->userq_mutex); 97 } 98 99 int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, 100 struct amdgpu_bo *bo) 101 { 102 struct dma_fence *ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr); 103 struct ttm_operation_ctx ctx = { false, false }; 104 struct dma_resv *resv = bo->tbo.base.resv; 105 int ret; 106 107 if (!dma_fence_is_signaled(ev_fence)) { 108 109 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); 110 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 111 if (!ret) 112 dma_resv_add_fence(resv, ev_fence, 113 DMA_RESV_USAGE_BOOKKEEP); 114 } else { 115 ret = 0; 116 } 117 118 dma_fence_put(ev_fence); 119 return ret; 120 } 121 122 int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr, 123 struct drm_exec *exec) 124 { 125 struct amdgpu_eviction_fence *ev_fence; 126 struct drm_gem_object *obj; 127 128 /* Create and initialize a new eviction fence */ 129 ev_fence = kzalloc_obj(*ev_fence); 130 if (!ev_fence) 131 return -ENOMEM; 132 133 ev_fence->evf_mgr = evf_mgr; 134 get_task_comm(ev_fence->timeline_name, current); 135 spin_lock_init(&ev_fence->lock); 136 dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops, 137 &ev_fence->lock, evf_mgr->ev_fence_ctx, 138 atomic_inc_return(&evf_mgr->ev_fence_seq)); 139 140 /* Remember it for newly added BOs */ 141 dma_fence_put(evf_mgr->ev_fence); 142 evf_mgr->ev_fence = &ev_fence->base; 143 144 /* And add it to all existing BOs */ 145 drm_exec_for_each_locked_object(exec, obj) { 146 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 147 148 amdgpu_evf_mgr_attach_fence(evf_mgr, bo); 149 } 150 return 0; 151 } 152 153 void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, 154 struct amdgpu_bo *bo) 155 { 156 struct dma_fence *stub = dma_fence_get_stub(); 157 158 dma_resv_replace_fences(bo->tbo.base.resv, evf_mgr->ev_fence_ctx, 159 stub, DMA_RESV_USAGE_BOOKKEEP); 160 dma_fence_put(stub); 161 } 162 163 void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr) 164 { 165 atomic_set(&evf_mgr->ev_fence_seq, 0); 166 evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1); 167 evf_mgr->ev_fence = dma_fence_get_stub(); 168 169 INIT_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker); 170 } 171 172 void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr) 173 { 174 evf_mgr->shutdown = true; 175 /* Make sure that the shutdown is visible to the suspend work */ 176 flush_work(&evf_mgr->suspend_work); 177 } 178 179 void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr) 180 { 181 dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true), 182 false); 183 /* Make sure that we are done with the last suspend work */ 184 flush_work(&evf_mgr->suspend_work); 185 } 186 187 void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr) 188 { 189 dma_fence_put(evf_mgr->ev_fence); 190 } 191