1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /* Copyright 2017-2019 Qiang Yu <yuq825@gmail.com> */
3
4 #include <linux/module.h>
5 #include <linux/of.h>
6 #include <linux/platform_device.h>
7 #include <linux/uaccess.h>
8 #include <linux/slab.h>
9 #include <linux/pm_runtime.h>
10 #include <drm/drm_ioctl.h>
11 #include <drm/drm_drv.h>
12 #include <drm/drm_prime.h>
13 #include <drm/lima_drm.h>
14
15 #include "lima_device.h"
16 #include "lima_drv.h"
17 #include "lima_gem.h"
18 #include "lima_vm.h"
19
20 int lima_sched_timeout_ms;
21 uint lima_heap_init_nr_pages = 8;
22 uint lima_max_error_tasks;
23 uint lima_job_hang_limit;
24
25 MODULE_PARM_DESC(sched_timeout_ms, "task run timeout in ms");
26 module_param_named(sched_timeout_ms, lima_sched_timeout_ms, int, 0444);
27
28 MODULE_PARM_DESC(heap_init_nr_pages, "heap buffer init number of pages");
29 module_param_named(heap_init_nr_pages, lima_heap_init_nr_pages, uint, 0444);
30
31 MODULE_PARM_DESC(max_error_tasks, "max number of error tasks to save");
32 module_param_named(max_error_tasks, lima_max_error_tasks, uint, 0644);
33
34 MODULE_PARM_DESC(job_hang_limit, "number of times to allow a job to hang before dropping it (default 0)");
35 module_param_named(job_hang_limit, lima_job_hang_limit, uint, 0444);
36
lima_ioctl_get_param(struct drm_device * dev,void * data,struct drm_file * file)37 static int lima_ioctl_get_param(struct drm_device *dev, void *data, struct drm_file *file)
38 {
39 struct drm_lima_get_param *args = data;
40 struct lima_device *ldev = to_lima_dev(dev);
41
42 if (args->pad)
43 return -EINVAL;
44
45 switch (args->param) {
46 case DRM_LIMA_PARAM_GPU_ID:
47 switch (ldev->id) {
48 case lima_gpu_mali400:
49 args->value = DRM_LIMA_PARAM_GPU_ID_MALI400;
50 break;
51 case lima_gpu_mali450:
52 args->value = DRM_LIMA_PARAM_GPU_ID_MALI450;
53 break;
54 default:
55 args->value = DRM_LIMA_PARAM_GPU_ID_UNKNOWN;
56 break;
57 }
58 break;
59
60 case DRM_LIMA_PARAM_NUM_PP:
61 args->value = ldev->pipe[lima_pipe_pp].num_processor;
62 break;
63
64 case DRM_LIMA_PARAM_GP_VERSION:
65 args->value = ldev->gp_version;
66 break;
67
68 case DRM_LIMA_PARAM_PP_VERSION:
69 args->value = ldev->pp_version;
70 break;
71
72 default:
73 return -EINVAL;
74 }
75
76 return 0;
77 }
78
lima_ioctl_gem_create(struct drm_device * dev,void * data,struct drm_file * file)79 static int lima_ioctl_gem_create(struct drm_device *dev, void *data, struct drm_file *file)
80 {
81 struct drm_lima_gem_create *args = data;
82
83 if (args->pad)
84 return -EINVAL;
85
86 if (args->flags & ~(LIMA_BO_FLAG_HEAP))
87 return -EINVAL;
88
89 if (args->size == 0)
90 return -EINVAL;
91
92 return lima_gem_create_handle(dev, file, args->size, args->flags, &args->handle);
93 }
94
lima_ioctl_gem_info(struct drm_device * dev,void * data,struct drm_file * file)95 static int lima_ioctl_gem_info(struct drm_device *dev, void *data, struct drm_file *file)
96 {
97 struct drm_lima_gem_info *args = data;
98
99 return lima_gem_get_info(file, args->handle, &args->va, &args->offset);
100 }
101
lima_ioctl_gem_submit(struct drm_device * dev,void * data,struct drm_file * file)102 static int lima_ioctl_gem_submit(struct drm_device *dev, void *data, struct drm_file *file)
103 {
104 struct drm_lima_gem_submit *args = data;
105 struct lima_device *ldev = to_lima_dev(dev);
106 struct lima_drm_priv *priv = file->driver_priv;
107 struct drm_lima_gem_submit_bo *bos;
108 struct lima_sched_pipe *pipe;
109 struct lima_sched_task *task;
110 struct lima_ctx *ctx;
111 struct lima_submit submit = {0};
112 size_t size;
113 int err = 0;
114
115 if (args->pipe >= lima_pipe_num || args->nr_bos == 0)
116 return -EINVAL;
117
118 if (args->flags & ~(LIMA_SUBMIT_FLAG_EXPLICIT_FENCE))
119 return -EINVAL;
120
121 pipe = ldev->pipe + args->pipe;
122 if (args->frame_size != pipe->frame_size)
123 return -EINVAL;
124
125 bos = kvcalloc(args->nr_bos, sizeof(*submit.bos) + sizeof(*submit.lbos), GFP_KERNEL);
126 if (!bos)
127 return -ENOMEM;
128
129 size = args->nr_bos * sizeof(*submit.bos);
130 if (copy_from_user(bos, u64_to_user_ptr(args->bos), size)) {
131 err = -EFAULT;
132 goto out0;
133 }
134
135 task = kmem_cache_zalloc(pipe->task_slab, GFP_KERNEL);
136 if (!task) {
137 err = -ENOMEM;
138 goto out0;
139 }
140
141 task->frame = task + 1;
142 if (copy_from_user(task->frame, u64_to_user_ptr(args->frame), args->frame_size)) {
143 err = -EFAULT;
144 goto out1;
145 }
146
147 err = pipe->task_validate(pipe, task);
148 if (err)
149 goto out1;
150
151 ctx = lima_ctx_get(&priv->ctx_mgr, args->ctx);
152 if (!ctx) {
153 err = -ENOENT;
154 goto out1;
155 }
156
157 submit.pipe = args->pipe;
158 submit.bos = bos;
159 submit.lbos = (void *)bos + size;
160 submit.nr_bos = args->nr_bos;
161 submit.task = task;
162 submit.ctx = ctx;
163 submit.flags = args->flags;
164 submit.in_sync[0] = args->in_sync[0];
165 submit.in_sync[1] = args->in_sync[1];
166 submit.out_sync = args->out_sync;
167
168 err = lima_gem_submit(file, &submit);
169
170 lima_ctx_put(ctx);
171 out1:
172 if (err)
173 kmem_cache_free(pipe->task_slab, task);
174 out0:
175 kvfree(bos);
176 return err;
177 }
178
lima_ioctl_gem_wait(struct drm_device * dev,void * data,struct drm_file * file)179 static int lima_ioctl_gem_wait(struct drm_device *dev, void *data, struct drm_file *file)
180 {
181 struct drm_lima_gem_wait *args = data;
182
183 if (args->op & ~(LIMA_GEM_WAIT_READ|LIMA_GEM_WAIT_WRITE))
184 return -EINVAL;
185
186 return lima_gem_wait(file, args->handle, args->op, args->timeout_ns);
187 }
188
lima_ioctl_ctx_create(struct drm_device * dev,void * data,struct drm_file * file)189 static int lima_ioctl_ctx_create(struct drm_device *dev, void *data, struct drm_file *file)
190 {
191 struct drm_lima_ctx_create *args = data;
192 struct lima_drm_priv *priv = file->driver_priv;
193 struct lima_device *ldev = to_lima_dev(dev);
194
195 if (args->_pad)
196 return -EINVAL;
197
198 return lima_ctx_create(ldev, &priv->ctx_mgr, &args->id);
199 }
200
lima_ioctl_ctx_free(struct drm_device * dev,void * data,struct drm_file * file)201 static int lima_ioctl_ctx_free(struct drm_device *dev, void *data, struct drm_file *file)
202 {
203 struct drm_lima_ctx_create *args = data;
204 struct lima_drm_priv *priv = file->driver_priv;
205
206 if (args->_pad)
207 return -EINVAL;
208
209 return lima_ctx_free(&priv->ctx_mgr, args->id);
210 }
211
lima_drm_driver_open(struct drm_device * dev,struct drm_file * file)212 static int lima_drm_driver_open(struct drm_device *dev, struct drm_file *file)
213 {
214 int err;
215 struct lima_drm_priv *priv;
216 struct lima_device *ldev = to_lima_dev(dev);
217
218 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
219 if (!priv)
220 return -ENOMEM;
221
222 priv->vm = lima_vm_create(ldev);
223 if (!priv->vm) {
224 err = -ENOMEM;
225 goto err_out0;
226 }
227
228 lima_ctx_mgr_init(&priv->ctx_mgr);
229
230 file->driver_priv = priv;
231 return 0;
232
233 err_out0:
234 kfree(priv);
235 return err;
236 }
237
lima_drm_driver_postclose(struct drm_device * dev,struct drm_file * file)238 static void lima_drm_driver_postclose(struct drm_device *dev, struct drm_file *file)
239 {
240 struct lima_drm_priv *priv = file->driver_priv;
241
242 lima_ctx_mgr_fini(&priv->ctx_mgr);
243 lima_vm_put(priv->vm);
244 kfree(priv);
245 }
246
247 static const struct drm_ioctl_desc lima_drm_driver_ioctls[] = {
248 DRM_IOCTL_DEF_DRV(LIMA_GET_PARAM, lima_ioctl_get_param, DRM_RENDER_ALLOW),
249 DRM_IOCTL_DEF_DRV(LIMA_GEM_CREATE, lima_ioctl_gem_create, DRM_RENDER_ALLOW),
250 DRM_IOCTL_DEF_DRV(LIMA_GEM_INFO, lima_ioctl_gem_info, DRM_RENDER_ALLOW),
251 DRM_IOCTL_DEF_DRV(LIMA_GEM_SUBMIT, lima_ioctl_gem_submit, DRM_RENDER_ALLOW),
252 DRM_IOCTL_DEF_DRV(LIMA_GEM_WAIT, lima_ioctl_gem_wait, DRM_RENDER_ALLOW),
253 DRM_IOCTL_DEF_DRV(LIMA_CTX_CREATE, lima_ioctl_ctx_create, DRM_RENDER_ALLOW),
254 DRM_IOCTL_DEF_DRV(LIMA_CTX_FREE, lima_ioctl_ctx_free, DRM_RENDER_ALLOW),
255 };
256
257 DEFINE_DRM_GEM_FOPS(lima_drm_driver_fops);
258
259 /*
260 * Changelog:
261 *
262 * - 1.1.0 - add heap buffer support
263 */
264
265 static const struct drm_driver lima_drm_driver = {
266 .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ,
267 .open = lima_drm_driver_open,
268 .postclose = lima_drm_driver_postclose,
269 .ioctls = lima_drm_driver_ioctls,
270 .num_ioctls = ARRAY_SIZE(lima_drm_driver_ioctls),
271 .fops = &lima_drm_driver_fops,
272 .name = "lima",
273 .desc = "lima DRM",
274 .date = "20191231",
275 .major = 1,
276 .minor = 1,
277 .patchlevel = 0,
278
279 .gem_create_object = lima_gem_create_object,
280 .gem_prime_import_sg_table = drm_gem_shmem_prime_import_sg_table,
281 };
282
283 struct lima_block_reader {
284 void *dst;
285 size_t base;
286 size_t count;
287 size_t off;
288 ssize_t read;
289 };
290
lima_read_block(struct lima_block_reader * reader,void * src,size_t src_size)291 static bool lima_read_block(struct lima_block_reader *reader,
292 void *src, size_t src_size)
293 {
294 size_t max_off = reader->base + src_size;
295
296 if (reader->off < max_off) {
297 size_t size = min_t(size_t, max_off - reader->off,
298 reader->count);
299
300 memcpy(reader->dst, src + (reader->off - reader->base), size);
301
302 reader->dst += size;
303 reader->off += size;
304 reader->read += size;
305 reader->count -= size;
306 }
307
308 reader->base = max_off;
309
310 return !!reader->count;
311 }
312
lima_error_state_read(struct file * filp,struct kobject * kobj,struct bin_attribute * attr,char * buf,loff_t off,size_t count)313 static ssize_t lima_error_state_read(struct file *filp, struct kobject *kobj,
314 struct bin_attribute *attr, char *buf,
315 loff_t off, size_t count)
316 {
317 struct device *dev = kobj_to_dev(kobj);
318 struct lima_device *ldev = dev_get_drvdata(dev);
319 struct lima_sched_error_task *et;
320 struct lima_block_reader reader = {
321 .dst = buf,
322 .count = count,
323 .off = off,
324 };
325
326 mutex_lock(&ldev->error_task_list_lock);
327
328 if (lima_read_block(&reader, &ldev->dump, sizeof(ldev->dump))) {
329 list_for_each_entry(et, &ldev->error_task_list, list) {
330 if (!lima_read_block(&reader, et->data, et->size))
331 break;
332 }
333 }
334
335 mutex_unlock(&ldev->error_task_list_lock);
336 return reader.read;
337 }
338
lima_error_state_write(struct file * file,struct kobject * kobj,struct bin_attribute * attr,char * buf,loff_t off,size_t count)339 static ssize_t lima_error_state_write(struct file *file, struct kobject *kobj,
340 struct bin_attribute *attr, char *buf,
341 loff_t off, size_t count)
342 {
343 struct device *dev = kobj_to_dev(kobj);
344 struct lima_device *ldev = dev_get_drvdata(dev);
345 struct lima_sched_error_task *et, *tmp;
346
347 mutex_lock(&ldev->error_task_list_lock);
348
349 list_for_each_entry_safe(et, tmp, &ldev->error_task_list, list) {
350 list_del(&et->list);
351 kvfree(et);
352 }
353
354 ldev->dump.size = 0;
355 ldev->dump.num_tasks = 0;
356
357 mutex_unlock(&ldev->error_task_list_lock);
358
359 return count;
360 }
361
362 static const struct bin_attribute lima_error_state_attr = {
363 .attr.name = "error",
364 .attr.mode = 0600,
365 .size = 0,
366 .read = lima_error_state_read,
367 .write = lima_error_state_write,
368 };
369
lima_pdev_probe(struct platform_device * pdev)370 static int lima_pdev_probe(struct platform_device *pdev)
371 {
372 struct lima_device *ldev;
373 struct drm_device *ddev;
374 const struct lima_compatible *comp;
375 int err;
376
377 err = lima_sched_slab_init();
378 if (err)
379 return err;
380
381 ldev = devm_kzalloc(&pdev->dev, sizeof(*ldev), GFP_KERNEL);
382 if (!ldev) {
383 err = -ENOMEM;
384 goto err_out0;
385 }
386
387 ldev->dev = &pdev->dev;
388 comp = of_device_get_match_data(&pdev->dev);
389 if (!comp) {
390 err = -ENODEV;
391 goto err_out0;
392 }
393
394 ldev->id = comp->id;
395
396 platform_set_drvdata(pdev, ldev);
397
398 /* Allocate and initialize the DRM device. */
399 ddev = drm_dev_alloc(&lima_drm_driver, &pdev->dev);
400 if (IS_ERR(ddev)) {
401 err = PTR_ERR(ddev);
402 goto err_out0;
403 }
404
405 ddev->dev_private = ldev;
406 ldev->ddev = ddev;
407
408 err = lima_device_init(ldev);
409 if (err)
410 goto err_out1;
411
412 err = lima_devfreq_init(ldev);
413 if (err) {
414 dev_err(&pdev->dev, "Fatal error during devfreq init\n");
415 goto err_out2;
416 }
417
418 pm_runtime_set_active(ldev->dev);
419 pm_runtime_mark_last_busy(ldev->dev);
420 pm_runtime_set_autosuspend_delay(ldev->dev, 200);
421 pm_runtime_use_autosuspend(ldev->dev);
422 pm_runtime_enable(ldev->dev);
423
424 /*
425 * Register the DRM device with the core and the connectors with
426 * sysfs.
427 */
428 err = drm_dev_register(ddev, 0);
429 if (err < 0)
430 goto err_out3;
431
432 if (sysfs_create_bin_file(&ldev->dev->kobj, &lima_error_state_attr))
433 dev_warn(ldev->dev, "fail to create error state sysfs\n");
434
435 return 0;
436
437 err_out3:
438 pm_runtime_disable(ldev->dev);
439 lima_devfreq_fini(ldev);
440 err_out2:
441 lima_device_fini(ldev);
442 err_out1:
443 drm_dev_put(ddev);
444 err_out0:
445 lima_sched_slab_fini();
446 return err;
447 }
448
lima_pdev_remove(struct platform_device * pdev)449 static void lima_pdev_remove(struct platform_device *pdev)
450 {
451 struct lima_device *ldev = platform_get_drvdata(pdev);
452 struct drm_device *ddev = ldev->ddev;
453
454 sysfs_remove_bin_file(&ldev->dev->kobj, &lima_error_state_attr);
455
456 drm_dev_unregister(ddev);
457
458 /* stop autosuspend to make sure device is in active state */
459 pm_runtime_set_autosuspend_delay(ldev->dev, -1);
460 pm_runtime_disable(ldev->dev);
461
462 lima_devfreq_fini(ldev);
463 lima_device_fini(ldev);
464
465 drm_dev_put(ddev);
466 lima_sched_slab_fini();
467 }
468
469 static const struct lima_compatible lima_mali400_data = {
470 .id = lima_gpu_mali400,
471 };
472
473 static const struct lima_compatible lima_mali450_data = {
474 .id = lima_gpu_mali450,
475 };
476
477 static const struct of_device_id dt_match[] = {
478 { .compatible = "arm,mali-400", .data = &lima_mali400_data },
479 { .compatible = "arm,mali-450", .data = &lima_mali450_data },
480 {}
481 };
482 MODULE_DEVICE_TABLE(of, dt_match);
483
484 static const struct dev_pm_ops lima_pm_ops = {
485 SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume)
486 SET_RUNTIME_PM_OPS(lima_device_suspend, lima_device_resume, NULL)
487 };
488
489 static struct platform_driver lima_platform_driver = {
490 .probe = lima_pdev_probe,
491 .remove = lima_pdev_remove,
492 .driver = {
493 .name = "lima",
494 .pm = &lima_pm_ops,
495 .of_match_table = dt_match,
496 },
497 };
498
499 module_platform_driver(lima_platform_driver);
500
501 MODULE_AUTHOR("Lima Project Developers");
502 MODULE_DESCRIPTION("Lima DRM Driver");
503 MODULE_LICENSE("GPL v2");
504 MODULE_SOFTDEP("pre: governor_simpleondemand");
505