1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright 2024 Advanced Micro Devices, Inc. 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 * OTHER DEALINGS IN THE SOFTWARE. 22 * 23 */ 24 #include <linux/sched.h> 25 #include <drm/drm_exec.h> 26 #include "amdgpu.h" 27 28 static const char * 29 amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence) 30 { 31 return "amdgpu_eviction_fence"; 32 } 33 34 static const char * 35 amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f) 36 { 37 struct amdgpu_eviction_fence *ef; 38 39 ef = container_of(f, struct amdgpu_eviction_fence, base); 40 return ef->timeline_name; 41 } 42 43 static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f) 44 { 45 struct amdgpu_eviction_fence *ev_fence = to_ev_fence(f); 46 47 schedule_work(&ev_fence->evf_mgr->suspend_work); 48 return true; 49 } 50 51 static const struct dma_fence_ops amdgpu_eviction_fence_ops = { 52 .get_driver_name = amdgpu_eviction_fence_get_driver_name, 53 .get_timeline_name = amdgpu_eviction_fence_get_timeline_name, 54 .enable_signaling = amdgpu_eviction_fence_enable_signaling, 55 }; 56 57 static void 58 amdgpu_eviction_fence_suspend_worker(struct work_struct *work) 59 { 60 struct amdgpu_eviction_fence_mgr *evf_mgr = 61 container_of(work, struct amdgpu_eviction_fence_mgr, 62 suspend_work); 63 struct amdgpu_fpriv *fpriv = 64 container_of(evf_mgr, struct amdgpu_fpriv, evf_mgr); 65 struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr; 66 struct dma_fence *ev_fence; 67 bool cookie; 68 69 mutex_lock(&uq_mgr->userq_mutex); 70 71 /* 72 * This is intentionally after taking the userq_mutex since we do 73 * allocate memory while holding this lock, but only after ensuring that 74 * the eviction fence is signaled. 75 */ 76 cookie = dma_fence_begin_signalling(); 77 78 ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr); 79 amdgpu_userq_evict(uq_mgr); 80 81 /* 82 * Signaling the eviction fence must be done while holding the 83 * userq_mutex. Otherwise we won't resume the queues before issuing the 84 * next fence. 85 */ 86 dma_fence_signal(ev_fence); 87 dma_fence_end_signalling(cookie); 88 dma_fence_put(ev_fence); 89 90 if (!evf_mgr->shutdown) 91 schedule_delayed_work(&uq_mgr->resume_work, 0); 92 93 mutex_unlock(&uq_mgr->userq_mutex); 94 } 95 96 int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, 97 struct amdgpu_bo *bo) 98 { 99 struct dma_fence *ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr); 100 struct ttm_operation_ctx ctx = { false, false }; 101 struct dma_resv *resv = bo->tbo.base.resv; 102 int ret; 103 104 if (!dma_fence_is_signaled(ev_fence)) { 105 106 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains); 107 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 108 if (!ret) 109 dma_resv_add_fence(resv, ev_fence, 110 DMA_RESV_USAGE_BOOKKEEP); 111 } else { 112 ret = 0; 113 } 114 115 dma_fence_put(ev_fence); 116 return ret; 117 } 118 119 int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr, 120 struct drm_exec *exec) 121 { 122 struct amdgpu_eviction_fence *ev_fence; 123 struct drm_gem_object *obj; 124 125 /* Create and initialize a new eviction fence */ 126 ev_fence = kzalloc_obj(*ev_fence); 127 if (!ev_fence) 128 return -ENOMEM; 129 130 ev_fence->evf_mgr = evf_mgr; 131 get_task_comm(ev_fence->timeline_name, current); 132 spin_lock_init(&ev_fence->lock); 133 dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops, 134 &ev_fence->lock, evf_mgr->ev_fence_ctx, 135 atomic_inc_return(&evf_mgr->ev_fence_seq)); 136 137 /* Remember it for newly added BOs */ 138 dma_fence_put(evf_mgr->ev_fence); 139 evf_mgr->ev_fence = &ev_fence->base; 140 141 /* And add it to all existing BOs */ 142 drm_exec_for_each_locked_object(exec, obj) { 143 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 144 145 amdgpu_evf_mgr_attach_fence(evf_mgr, bo); 146 } 147 return 0; 148 } 149 150 void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr, 151 struct amdgpu_bo *bo) 152 { 153 struct dma_fence *stub = dma_fence_get_stub(); 154 155 dma_resv_replace_fences(bo->tbo.base.resv, evf_mgr->ev_fence_ctx, 156 stub, DMA_RESV_USAGE_BOOKKEEP); 157 dma_fence_put(stub); 158 } 159 160 void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr) 161 { 162 atomic_set(&evf_mgr->ev_fence_seq, 0); 163 evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1); 164 evf_mgr->ev_fence = dma_fence_get_stub(); 165 166 INIT_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker); 167 } 168 169 void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr) 170 { 171 evf_mgr->shutdown = true; 172 /* Make sure that the shutdown is visible to the suspend work */ 173 flush_work(&evf_mgr->suspend_work); 174 } 175 176 void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr) 177 { 178 dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true), 179 false); 180 /* Make sure that we are done with the last suspend work */ 181 flush_work(&evf_mgr->suspend_work); 182 } 183 184 void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr) 185 { 186 dma_fence_put(evf_mgr->ev_fence); 187 } 188