1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2024 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include <linux/sched.h> 25 #include <drm/drm_exec.h> 26 #include "amdgpu.h" 27 28 static const char * 29 amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence) 30 { 31 return "amdgpu_eviction_fence"; 32 } 33 34 static const char * 35 amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f) 36 { 37 struct amdgpu_eviction_fence *ef; 38 39 ef = container_of(f, struct amdgpu_eviction_fence, base); 40 return ef->timeline_name; 41 } 42 43 static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f) 44 { 45 struct amdgpu_eviction_fence *ev_fence = to_ev_fence(f); 46 47 schedule_work(&ev_fence->evf_mgr->suspend_work); 48 return true; 49 } 50 51 static const struct dma_fence_ops amdgpu_eviction_fence_ops = { 52 .get_driver_name = amdgpu_eviction_fence_get_driver_name, 53 .get_timeline_name = amdgpu_eviction_fence_get_timeline_name, 54 .enable_signaling = amdgpu_eviction_fence_enable_signaling, 55 }; 56 57 static void 58 amdgpu_eviction_fence_suspend_worker(struct work_struct *work) 59 { 60 struct amdgpu_eviction_fence_mgr *evf_mgr = 61 container_of(work, struct amdgpu_eviction_fence_mgr, 62 suspend_work); 63 struct amdgpu_fpriv *fpriv = 64 container_of(evf_mgr, struct amdgpu_fpriv, evf_mgr); 65 struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; 66 struct dma_fence *ev_fence; 67 bool cookie; 68 69 mutex_lock(&uq_mgr->userq_mutex); 70 71 /* 72 * This is intentionally after taking the userq_mutex since we do 73 * allocate memory while holding this lock, but only after ensuring that 74 * the eviction fence is signaled. 75 */ 76 cookie = dma_fence_begin_signalling(); 77 78 ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr); 79 amdgpu_userq_evict(uq_mgr); 80 81 /* 82 * Signaling the eviction fence must be done while holding the 83 * userq_mutex. Otherwise we won't resume the queues before issuing the 84 * next fence. 85 */ 86 dma_fence_signal(ev_fence); 87 dma_fence_end_signalling(cookie); 88 dma_fence_put(ev_fence); 89 90 if (!evf_mgr->shutdown) 91 schedule_delayed_work(&uq_mgr->resume_work, 0); 92 93 mutex_unlock(&uq_mgr->userq_mutex); 94 } 95 96 int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, 97 struct amdgpu_bo *bo) 98 { 99 struct dma_fence *ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr); 100 struct ttm_operation_ctx ctx = { false, false }; 101 struct dma_resv *resv = bo->tbo.base.resv; 102 int ret; 103 104 if (!dma_fence_is_signaled(ev_fence)) { 105 106 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); 107 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 108 if (!ret) 109 dma_resv_add_fence(resv, ev_fence, 110 DMA_RESV_USAGE_BOOKKEEP); 111 } else { 112 ret = 0; 113 } 114 115 dma_fence_put(ev_fence); 116 return ret; 117 } 118 119 int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr, 120 struct drm_exec *exec) 121 { 122 struct amdgpu_eviction_fence *ev_fence; 123 struct drm_gem_object *obj; 124 unsigned long index; 125 126 /* Create and initialize a new eviction fence */ 127 ev_fence = kzalloc_obj(*ev_fence); 128 if (!ev_fence) 129 return -ENOMEM; 130 131 ev_fence->evf_mgr = evf_mgr; 132 get_task_comm(ev_fence->timeline_name, current); 133 spin_lock_init(&ev_fence->lock); 134 dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops, 135 &ev_fence->lock, evf_mgr->ev_fence_ctx, 136 atomic_inc_return(&evf_mgr->ev_fence_seq)); 137 138 /* Remember it for newly added BOs */ 139 dma_fence_put(evf_mgr->ev_fence); 140 evf_mgr->ev_fence = &ev_fence->base; 141 142 /* And add it to all existing BOs */ 143 drm_exec_for_each_locked_object(exec, index, obj) { 144 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 145 146 amdgpu_evf_mgr_attach_fence(evf_mgr, bo); 147 } 148 return 0; 149 } 150 151 void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, 152 struct amdgpu_bo *bo) 153 { 154 struct dma_fence *stub = dma_fence_get_stub(); 155 156 dma_resv_replace_fences(bo->tbo.base.resv, evf_mgr->ev_fence_ctx, 157 stub, DMA_RESV_USAGE_BOOKKEEP); 158 dma_fence_put(stub); 159 } 160 161 void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr) 162 { 163 atomic_set(&evf_mgr->ev_fence_seq, 0); 164 evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1); 165 evf_mgr->ev_fence = dma_fence_get_stub(); 166 167 INIT_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker); 168 } 169 170 void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr) 171 { 172 evf_mgr->shutdown = true; 173 /* Make sure that the shutdown is visible to the suspend work */ 174 flush_work(&evf_mgr->suspend_work); 175 } 176 177 void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr) 178 { 179 dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true), 180 false); 181 /* Make sure that we are done with the last suspend work */ 182 flush_work(&evf_mgr->suspend_work); 183 } 184 185 void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr) 186 { 187 dma_fence_put(evf_mgr->ev_fence); 188 } 189