1 /*
2 * Copyright 2015 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26 /*
27 * Authors:
28 * Christian König <deathsimple@vodafone.de>
29 */
30
31 #include <linux/sort.h>
32 #include <linux/uaccess.h>
33
34 #include "amdgpu.h"
35 #include "amdgpu_trace.h"
36
37 #define AMDGPU_BO_LIST_MAX_PRIORITY 32u
38 #define AMDGPU_BO_LIST_NUM_BUCKETS (AMDGPU_BO_LIST_MAX_PRIORITY + 1)
39
amdgpu_bo_list_free_rcu(struct rcu_head * rcu)40 static void amdgpu_bo_list_free_rcu(struct rcu_head *rcu)
41 {
42 struct amdgpu_bo_list *list = container_of(rcu, struct amdgpu_bo_list,
43 rhead);
44 mutex_destroy(&list->bo_list_mutex);
45 kvfree(list);
46 }
47
amdgpu_bo_list_free(struct kref * ref)48 static void amdgpu_bo_list_free(struct kref *ref)
49 {
50 struct amdgpu_bo_list *list = container_of(ref, struct amdgpu_bo_list,
51 refcount);
52 struct amdgpu_bo_list_entry *e;
53
54 amdgpu_bo_list_for_each_entry(e, list)
55 amdgpu_bo_unref(&e->bo);
56 call_rcu(&list->rhead, amdgpu_bo_list_free_rcu);
57 }
58
amdgpu_bo_list_entry_cmp(const void * _a,const void * _b)59 static int amdgpu_bo_list_entry_cmp(const void *_a, const void *_b)
60 {
61 const struct amdgpu_bo_list_entry *a = _a, *b = _b;
62
63 BUILD_BUG_ON(AMDGPU_BO_LIST_MAX_PRIORITY >= INT_MAX);
64
65 return (int)a->priority - (int)b->priority;
66 }
67
amdgpu_bo_list_create(struct amdgpu_device * adev,struct drm_file * filp,struct drm_amdgpu_bo_list_entry * info,size_t num_entries,struct amdgpu_bo_list ** result)68 int amdgpu_bo_list_create(struct amdgpu_device *adev, struct drm_file *filp,
69 struct drm_amdgpu_bo_list_entry *info,
70 size_t num_entries, struct amdgpu_bo_list **result)
71 {
72 unsigned last_entry = 0, first_userptr = num_entries;
73 struct amdgpu_bo_list_entry *array;
74 struct amdgpu_bo_list *list;
75 uint64_t total_size = 0;
76 unsigned i;
77 int r;
78
79 list = kvzalloc_flex(*list, entries, num_entries);
80 if (!list)
81 return -ENOMEM;
82
83 kref_init(&list->refcount);
84
85 list->num_entries = num_entries;
86 array = list->entries;
87
88 for (i = 0; i < num_entries; ++i) {
89 struct amdgpu_bo_list_entry *entry;
90 struct drm_gem_object *gobj;
91 struct amdgpu_bo *bo;
92 struct mm_struct *usermm;
93
94 gobj = drm_gem_object_lookup(filp, info[i].bo_handle);
95 if (!gobj) {
96 r = -ENOENT;
97 goto error_free;
98 }
99
100 bo = amdgpu_bo_ref(gem_to_amdgpu_bo(gobj));
101 drm_gem_object_put(gobj);
102
103 usermm = amdgpu_ttm_tt_get_usermm(bo->tbo.ttm);
104 if (usermm) {
105 if (usermm != current->mm) {
106 amdgpu_bo_unref(&bo);
107 r = -EPERM;
108 goto error_free;
109 }
110 entry = &array[--first_userptr];
111 } else {
112 entry = &array[last_entry++];
113 }
114
115 entry->priority = min(info[i].bo_priority,
116 AMDGPU_BO_LIST_MAX_PRIORITY);
117 entry->bo = bo;
118
119 if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GDS)
120 list->gds_obj = bo;
121 if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_GWS)
122 list->gws_obj = bo;
123 if (bo->preferred_domains == AMDGPU_GEM_DOMAIN_OA)
124 list->oa_obj = bo;
125
126 total_size += amdgpu_bo_size(bo);
127 trace_amdgpu_bo_list_set(list, bo);
128 }
129
130 list->first_userptr = first_userptr;
131 sort(array, last_entry, sizeof(struct amdgpu_bo_list_entry),
132 amdgpu_bo_list_entry_cmp, NULL);
133
134 trace_amdgpu_cs_bo_status(list->num_entries, total_size);
135
136 mutex_init(&list->bo_list_mutex);
137 *result = list;
138 return 0;
139
140 error_free:
141 for (i = 0; i < last_entry; ++i)
142 amdgpu_bo_unref(&array[i].bo);
143 for (i = first_userptr; i < num_entries; ++i)
144 amdgpu_bo_unref(&array[i].bo);
145 kvfree(list);
146 return r;
147
148 }
149
amdgpu_bo_list_destroy(struct amdgpu_fpriv * fpriv,int id)150 static void amdgpu_bo_list_destroy(struct amdgpu_fpriv *fpriv, int id)
151 {
152 struct amdgpu_bo_list *list;
153
154 mutex_lock(&fpriv->bo_list_lock);
155 list = idr_remove(&fpriv->bo_list_handles, id);
156 mutex_unlock(&fpriv->bo_list_lock);
157 if (list)
158 kref_put(&list->refcount, amdgpu_bo_list_free);
159 }
160
amdgpu_bo_list_get(struct amdgpu_fpriv * fpriv,int id,struct amdgpu_bo_list ** result)161 int amdgpu_bo_list_get(struct amdgpu_fpriv *fpriv, int id,
162 struct amdgpu_bo_list **result)
163 {
164 rcu_read_lock();
165 *result = idr_find(&fpriv->bo_list_handles, id);
166
167 if (*result && kref_get_unless_zero(&(*result)->refcount)) {
168 rcu_read_unlock();
169 return 0;
170 }
171
172 rcu_read_unlock();
173 *result = NULL;
174 return -ENOENT;
175 }
176
amdgpu_bo_list_put(struct amdgpu_bo_list * list)177 void amdgpu_bo_list_put(struct amdgpu_bo_list *list)
178 {
179 kref_put(&list->refcount, amdgpu_bo_list_free);
180 }
181
amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in * in,struct drm_amdgpu_bo_list_entry ** info_param)182 int amdgpu_bo_create_list_entry_array(struct drm_amdgpu_bo_list_in *in,
183 struct drm_amdgpu_bo_list_entry **info_param)
184 {
185 const uint32_t info_size = sizeof(struct drm_amdgpu_bo_list_entry);
186 const void __user *uptr = u64_to_user_ptr(in->bo_info_ptr);
187 const uint32_t bo_info_size = in->bo_info_size;
188 const uint32_t bo_number = in->bo_number;
189 struct drm_amdgpu_bo_list_entry *info;
190
191 /* copy the handle array from userspace to a kernel buffer */
192 if (likely(info_size == bo_info_size)) {
193 info = vmemdup_array_user(uptr, bo_number, info_size);
194 if (IS_ERR(info))
195 return PTR_ERR(info);
196 } else {
197 const uint32_t bytes = min(bo_info_size, info_size);
198 unsigned i;
199
200 info = kvmalloc_array(bo_number, info_size, GFP_KERNEL);
201 if (!info)
202 return -ENOMEM;
203
204 memset(info, 0, bo_number * info_size);
205 for (i = 0; i < bo_number; ++i, uptr += bo_info_size) {
206 if (copy_from_user(&info[i], uptr, bytes)) {
207 kvfree(info);
208 return -EFAULT;
209 }
210 }
211 }
212
213 *info_param = info;
214 return 0;
215 }
216
amdgpu_bo_list_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)217 int amdgpu_bo_list_ioctl(struct drm_device *dev, void *data,
218 struct drm_file *filp)
219 {
220 struct amdgpu_device *adev = drm_to_adev(dev);
221 struct amdgpu_fpriv *fpriv = filp->driver_priv;
222 union drm_amdgpu_bo_list *args = data;
223 uint32_t handle = args->in.list_handle;
224 struct drm_amdgpu_bo_list_entry *info = NULL;
225 struct amdgpu_bo_list *list, *old;
226 int r;
227
228 r = amdgpu_bo_create_list_entry_array(&args->in, &info);
229 if (r)
230 return r;
231
232 switch (args->in.operation) {
233 case AMDGPU_BO_LIST_OP_CREATE:
234 r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number,
235 &list);
236 if (r)
237 goto error_free;
238
239 mutex_lock(&fpriv->bo_list_lock);
240 r = idr_alloc(&fpriv->bo_list_handles, list, 1, 0, GFP_KERNEL);
241 mutex_unlock(&fpriv->bo_list_lock);
242 if (r < 0) {
243 goto error_put_list;
244 }
245
246 handle = r;
247 break;
248
249 case AMDGPU_BO_LIST_OP_DESTROY:
250 amdgpu_bo_list_destroy(fpriv, handle);
251 handle = 0;
252 break;
253
254 case AMDGPU_BO_LIST_OP_UPDATE:
255 r = amdgpu_bo_list_create(adev, filp, info, args->in.bo_number,
256 &list);
257 if (r)
258 goto error_free;
259
260 mutex_lock(&fpriv->bo_list_lock);
261 old = idr_replace(&fpriv->bo_list_handles, list, handle);
262 mutex_unlock(&fpriv->bo_list_lock);
263
264 if (IS_ERR(old)) {
265 r = PTR_ERR(old);
266 goto error_put_list;
267 }
268
269 amdgpu_bo_list_put(old);
270 break;
271
272 default:
273 r = -EINVAL;
274 goto error_free;
275 }
276
277 memset(args, 0, sizeof(*args));
278 args->out.list_handle = handle;
279 kvfree(info);
280
281 return 0;
282
283 error_put_list:
284 amdgpu_bo_list_put(list);
285
286 error_free:
287 kvfree(info);
288 return r;
289 }
290