1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright 2024 Advanced Micro Devices, Inc.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
19 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
20 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
21 * OTHER DEALINGS IN THE SOFTWARE.
22 *
23 */
24 #include <linux/sched.h>
25 #include <drm/drm_exec.h>
26 #include "amdgpu.h"
27
28 static const char *
amdgpu_eviction_fence_get_driver_name(struct dma_fence * fence)29 amdgpu_eviction_fence_get_driver_name(struct dma_fence *fence)
30 {
31 return "amdgpu_eviction_fence";
32 }
33
34 static const char *
amdgpu_eviction_fence_get_timeline_name(struct dma_fence * f)35 amdgpu_eviction_fence_get_timeline_name(struct dma_fence *f)
36 {
37 struct amdgpu_eviction_fence *ef;
38
39 ef = container_of(f, struct amdgpu_eviction_fence, base);
40 return ef->timeline_name;
41 }
42
amdgpu_eviction_fence_enable_signaling(struct dma_fence * f)43 static bool amdgpu_eviction_fence_enable_signaling(struct dma_fence *f)
44 {
45 struct amdgpu_eviction_fence *ev_fence = to_ev_fence(f);
46
47 schedule_work(&ev_fence->evf_mgr->suspend_work);
48 return true;
49 }
50
51 static const struct dma_fence_ops amdgpu_eviction_fence_ops = {
52 .get_driver_name = amdgpu_eviction_fence_get_driver_name,
53 .get_timeline_name = amdgpu_eviction_fence_get_timeline_name,
54 .enable_signaling = amdgpu_eviction_fence_enable_signaling,
55 };
56
57 static void
amdgpu_eviction_fence_suspend_worker(struct work_struct * work)58 amdgpu_eviction_fence_suspend_worker(struct work_struct *work)
59 {
60 struct amdgpu_eviction_fence_mgr *evf_mgr =
61 container_of(work, struct amdgpu_eviction_fence_mgr,
62 suspend_work);
63 struct amdgpu_fpriv *fpriv =
64 container_of(evf_mgr, struct amdgpu_fpriv, evf_mgr);
65 struct amdgpu_userq_mgr *uq_mgr = &fpriv->userq_mgr;
66 struct dma_fence *ev_fence;
67 bool cookie;
68
69 mutex_lock(&uq_mgr->userq_mutex);
70
71 /* Fence waits are not allowed in a fence signalling critical section. */
72 amdgpu_userq_wait_for_signal(uq_mgr);
73
74 /*
75 * This is intentionally after taking the userq_mutex since we do
76 * allocate memory while holding this lock, but only after ensuring that
77 * the eviction fence is signaled.
78 */
79 cookie = dma_fence_begin_signalling();
80
81 ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
82 amdgpu_userq_evict(uq_mgr);
83
84 /*
85 * Signaling the eviction fence must be done while holding the
86 * userq_mutex. Otherwise we won't resume the queues before issuing the
87 * next fence.
88 */
89 dma_fence_signal(ev_fence);
90 dma_fence_end_signalling(cookie);
91 dma_fence_put(ev_fence);
92
93 if (!evf_mgr->shutdown)
94 schedule_delayed_work(&uq_mgr->resume_work, 0);
95
96 mutex_unlock(&uq_mgr->userq_mutex);
97 }
98
amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr * evf_mgr,struct amdgpu_bo * bo)99 int amdgpu_evf_mgr_attach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
100 struct amdgpu_bo *bo)
101 {
102 struct dma_fence *ev_fence = amdgpu_evf_mgr_get_fence(evf_mgr);
103 struct ttm_operation_ctx ctx = { false, false };
104 struct dma_resv *resv = bo->tbo.base.resv;
105 int ret;
106
107 if (!dma_fence_is_signaled(ev_fence)) {
108
109 amdgpu_bo_placement_from_domain(bo, bo->allowed_domains);
110 ret = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx);
111 if (!ret)
112 dma_resv_add_fence(resv, ev_fence,
113 DMA_RESV_USAGE_BOOKKEEP);
114 } else {
115 ret = 0;
116 }
117
118 dma_fence_put(ev_fence);
119 return ret;
120 }
121
amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr * evf_mgr,struct drm_exec * exec)122 int amdgpu_evf_mgr_rearm(struct amdgpu_eviction_fence_mgr *evf_mgr,
123 struct drm_exec *exec)
124 {
125 struct amdgpu_eviction_fence *ev_fence;
126 struct drm_gem_object *obj;
127
128 /* Create and initialize a new eviction fence */
129 ev_fence = kzalloc_obj(*ev_fence);
130 if (!ev_fence)
131 return -ENOMEM;
132
133 ev_fence->evf_mgr = evf_mgr;
134 get_task_comm(ev_fence->timeline_name, current);
135 spin_lock_init(&ev_fence->lock);
136 dma_fence_init64(&ev_fence->base, &amdgpu_eviction_fence_ops,
137 &ev_fence->lock, evf_mgr->ev_fence_ctx,
138 atomic_inc_return(&evf_mgr->ev_fence_seq));
139
140 /* Remember it for newly added BOs */
141 dma_fence_put(evf_mgr->ev_fence);
142 evf_mgr->ev_fence = &ev_fence->base;
143
144 /* And add it to all existing BOs */
145 drm_exec_for_each_locked_object(exec, obj) {
146 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj);
147
148 amdgpu_evf_mgr_attach_fence(evf_mgr, bo);
149 }
150 return 0;
151 }
152
amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr * evf_mgr,struct amdgpu_bo * bo)153 void amdgpu_evf_mgr_detach_fence(struct amdgpu_eviction_fence_mgr *evf_mgr,
154 struct amdgpu_bo *bo)
155 {
156 struct dma_fence *stub = dma_fence_get_stub();
157
158 dma_resv_replace_fences(bo->tbo.base.resv, evf_mgr->ev_fence_ctx,
159 stub, DMA_RESV_USAGE_BOOKKEEP);
160 dma_fence_put(stub);
161 }
162
amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr * evf_mgr)163 void amdgpu_evf_mgr_init(struct amdgpu_eviction_fence_mgr *evf_mgr)
164 {
165 atomic_set(&evf_mgr->ev_fence_seq, 0);
166 evf_mgr->ev_fence_ctx = dma_fence_context_alloc(1);
167 evf_mgr->ev_fence = dma_fence_get_stub();
168
169 INIT_WORK(&evf_mgr->suspend_work, amdgpu_eviction_fence_suspend_worker);
170 }
171
amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr * evf_mgr)172 void amdgpu_evf_mgr_shutdown(struct amdgpu_eviction_fence_mgr *evf_mgr)
173 {
174 evf_mgr->shutdown = true;
175 /* Make sure that the shutdown is visible to the suspend work */
176 flush_work(&evf_mgr->suspend_work);
177 }
178
amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr * evf_mgr)179 void amdgpu_evf_mgr_flush_suspend(struct amdgpu_eviction_fence_mgr *evf_mgr)
180 {
181 dma_fence_wait(rcu_dereference_protected(evf_mgr->ev_fence, true),
182 false);
183 /* Make sure that we are done with the last suspend work */
184 flush_work(&evf_mgr->suspend_work);
185 }
186
amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr * evf_mgr)187 void amdgpu_evf_mgr_fini(struct amdgpu_eviction_fence_mgr *evf_mgr)
188 {
189 dma_fence_put(evf_mgr->ev_fence);
190 }
191