1 // SPDX-License-Identifier: GPL-2.0-only or MIT
2 /* Copyright 2025 Arm, Ltd. */
3
4 #include <linux/err.h>
5 #include <linux/slab.h>
6
7 #include <drm/ethosu_accel.h>
8
9 #include "ethosu_device.h"
10 #include "ethosu_gem.h"
11
ethosu_gem_free_object(struct drm_gem_object * obj)12 static void ethosu_gem_free_object(struct drm_gem_object *obj)
13 {
14 struct ethosu_gem_object *bo = to_ethosu_bo(obj);
15
16 kfree(bo->info);
17 drm_gem_free_mmap_offset(&bo->base.base);
18 drm_gem_dma_free(&bo->base);
19 }
20
ethosu_gem_mmap(struct drm_gem_object * obj,struct vm_area_struct * vma)21 static int ethosu_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
22 {
23 struct ethosu_gem_object *bo = to_ethosu_bo(obj);
24
25 /* Don't allow mmap on objects that have the NO_MMAP flag set. */
26 if (bo->flags & DRM_ETHOSU_BO_NO_MMAP)
27 return -EINVAL;
28
29 return drm_gem_dma_object_mmap(obj, vma);
30 }
31
32 static const struct drm_gem_object_funcs ethosu_gem_funcs = {
33 .free = ethosu_gem_free_object,
34 .print_info = drm_gem_dma_object_print_info,
35 .get_sg_table = drm_gem_dma_object_get_sg_table,
36 .vmap = drm_gem_dma_object_vmap,
37 .mmap = ethosu_gem_mmap,
38 .vm_ops = &drm_gem_dma_vm_ops,
39 };
40
41 /**
42 * ethosu_gem_create_object - Implementation of driver->gem_create_object.
43 * @ddev: DRM device
44 * @size: Size in bytes of the memory the object will reference
45 *
46 * This lets the GEM helpers allocate object structs for us, and keep
47 * our BO stats correct.
48 */
ethosu_gem_create_object(struct drm_device * ddev,size_t size)49 struct drm_gem_object *ethosu_gem_create_object(struct drm_device *ddev, size_t size)
50 {
51 struct ethosu_gem_object *obj;
52
53 obj = kzalloc_obj(*obj);
54 if (!obj)
55 return ERR_PTR(-ENOMEM);
56
57 obj->base.base.funcs = ðosu_gem_funcs;
58 return &obj->base.base;
59 }
60
61 /**
62 * ethosu_gem_create_with_handle() - Create a GEM object and attach it to a handle.
63 * @file: DRM file.
64 * @ddev: DRM device.
65 * @size: Size of the GEM object to allocate.
66 * @flags: Combination of drm_ethosu_bo_flags flags.
67 * @handle: Pointer holding the handle pointing to the new GEM object.
68 *
69 * Return: Zero on success
70 */
ethosu_gem_create_with_handle(struct drm_file * file,struct drm_device * ddev,u64 * size,u32 flags,u32 * handle)71 int ethosu_gem_create_with_handle(struct drm_file *file,
72 struct drm_device *ddev,
73 u64 *size, u32 flags, u32 *handle)
74 {
75 struct drm_gem_dma_object *mem;
76 struct ethosu_gem_object *bo;
77 int ret;
78
79 mem = drm_gem_dma_create(ddev, *size);
80 if (IS_ERR(mem))
81 return PTR_ERR(mem);
82
83 bo = to_ethosu_bo(&mem->base);
84 bo->flags = flags;
85
86 /*
87 * Allocate an id of idr table where the obj is registered
88 * and handle has the id what user can see.
89 */
90 ret = drm_gem_handle_create(file, &mem->base, handle);
91 if (!ret)
92 *size = bo->base.base.size;
93
94 /* drop reference from allocate - handle holds it now. */
95 drm_gem_object_put(&mem->base);
96
97 return ret;
98 }
99
100 struct dma {
101 s8 region;
102 u64 len;
103 u64 offset;
104 s64 stride[2];
105 };
106
107 struct dma_state {
108 u16 size0;
109 u16 size1;
110 s8 mode;
111 struct dma src;
112 struct dma dst;
113 };
114
115 struct buffer {
116 u64 base;
117 u32 length;
118 s8 region;
119 };
120
121 struct feat_matrix {
122 u64 base[4];
123 s64 stride_x;
124 s64 stride_y;
125 s64 stride_c;
126 s8 region;
127 u8 broadcast;
128 u16 stride_kernel;
129 u16 precision;
130 u16 depth;
131 u16 width;
132 u16 width0;
133 u16 height[3];
134 u8 pad_top;
135 u8 pad_left;
136 u8 pad_bottom;
137 u8 pad_right;
138 };
139
140 struct cmd_state {
141 struct dma_state dma;
142 struct buffer scale[2];
143 struct buffer weight[4];
144 struct feat_matrix ofm;
145 struct feat_matrix ifm;
146 struct feat_matrix ifm2;
147 };
148
cmd_state_init(struct cmd_state * st)149 static void cmd_state_init(struct cmd_state *st)
150 {
151 /* Initialize to all 1s to detect missing setup */
152 memset(st, 0xff, sizeof(*st));
153 }
154
cmd_to_addr(u32 * cmd)155 static u64 cmd_to_addr(u32 *cmd)
156 {
157 return (((u64)cmd[0] & 0xff0000) << 16) | cmd[1];
158 }
159
dma_length(struct ethosu_validated_cmdstream_info * info,struct dma_state * dma_st,struct dma * dma)160 static u64 dma_length(struct ethosu_validated_cmdstream_info *info,
161 struct dma_state *dma_st, struct dma *dma)
162 {
163 s8 mode = dma_st->mode;
164 u64 len = dma->len;
165
166 if (mode >= 1) {
167 len += dma->stride[0];
168 len *= dma_st->size0;
169 }
170 if (mode == 2) {
171 len += dma->stride[1];
172 len *= dma_st->size1;
173 }
174 if (dma->region >= 0)
175 info->region_size[dma->region] = max(info->region_size[dma->region],
176 len + dma->offset);
177
178 return len;
179 }
180
feat_matrix_length(struct ethosu_validated_cmdstream_info * info,struct feat_matrix * fm,u32 x,u32 y,u32 c)181 static u64 feat_matrix_length(struct ethosu_validated_cmdstream_info *info,
182 struct feat_matrix *fm,
183 u32 x, u32 y, u32 c)
184 {
185 u32 element_size, storage = fm->precision >> 14;
186 int tile = 0;
187 u64 addr;
188
189 if (fm->region < 0)
190 return U64_MAX;
191
192 switch (storage) {
193 case 0:
194 if (x >= fm->width0 + 1) {
195 x -= fm->width0 + 1;
196 tile += 1;
197 }
198 if (y >= fm->height[tile] + 1) {
199 y -= fm->height[tile] + 1;
200 tile += 2;
201 }
202 break;
203 case 1:
204 if (y >= fm->height[1] + 1) {
205 y -= fm->height[1] + 1;
206 tile = 2;
207 } else if (y >= fm->height[0] + 1) {
208 y -= fm->height[0] + 1;
209 tile = 1;
210 }
211 break;
212 }
213 if (fm->base[tile] == U64_MAX)
214 return U64_MAX;
215
216 addr = fm->base[tile] + y * fm->stride_y;
217
218 switch ((fm->precision >> 6) & 0x3) { // format
219 case 0: //nhwc:
220 addr += x * fm->stride_x + c;
221 break;
222 case 1: //nhcwb16:
223 element_size = BIT((fm->precision >> 1) & 0x3);
224
225 addr += (c / 16) * fm->stride_c + (16 * x + (c & 0xf)) * element_size;
226 break;
227 }
228
229 info->region_size[fm->region] = max(info->region_size[fm->region], addr + 1);
230
231 return addr;
232 }
233
calc_sizes(struct drm_device * ddev,struct ethosu_validated_cmdstream_info * info,u16 op,struct cmd_state * st,bool ifm,bool ifm2,bool weight,bool scale)234 static int calc_sizes(struct drm_device *ddev,
235 struct ethosu_validated_cmdstream_info *info,
236 u16 op, struct cmd_state *st,
237 bool ifm, bool ifm2, bool weight, bool scale)
238 {
239 u64 len;
240
241 if (ifm) {
242 if (st->ifm.stride_kernel == U16_MAX)
243 return -EINVAL;
244 u32 stride_y = ((st->ifm.stride_kernel >> 8) & 0x2) +
245 ((st->ifm.stride_kernel >> 1) & 0x1) + 1;
246 u32 stride_x = ((st->ifm.stride_kernel >> 5) & 0x2) +
247 (st->ifm.stride_kernel & 0x1) + 1;
248 s32 ifm_height = st->ofm.height[2] * stride_y +
249 st->ifm.height[2] - (st->ifm.pad_top + st->ifm.pad_bottom);
250 s32 ifm_width = st->ofm.width * stride_x +
251 st->ifm.width - (st->ifm.pad_left + st->ifm.pad_right);
252
253 if (ifm_height < 0 || ifm_width < 0)
254 return -EINVAL;
255
256 len = feat_matrix_length(info, &st->ifm, ifm_width,
257 ifm_height, st->ifm.depth);
258 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n",
259 op, st->ifm.region, st->ifm.base[0], len);
260 if (len == U64_MAX)
261 return -EINVAL;
262 }
263
264 if (ifm2) {
265 len = feat_matrix_length(info, &st->ifm2, st->ifm.depth,
266 0, st->ofm.depth);
267 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n",
268 op, st->ifm2.region, st->ifm2.base[0], len);
269 if (len == U64_MAX)
270 return -EINVAL;
271 }
272
273 if (weight) {
274 dev_dbg(ddev->dev, "op %d: W:%d:0x%llx-0x%llx\n",
275 op, st->weight[0].region, st->weight[0].base,
276 st->weight[0].base + st->weight[0].length - 1);
277 if (st->weight[0].region < 0 || st->weight[0].base == U64_MAX ||
278 st->weight[0].length == U32_MAX)
279 return -EINVAL;
280 info->region_size[st->weight[0].region] =
281 max(info->region_size[st->weight[0].region],
282 st->weight[0].base + st->weight[0].length);
283 }
284
285 if (scale) {
286 dev_dbg(ddev->dev, "op %d: S:%d:0x%llx-0x%llx\n",
287 op, st->scale[0].region, st->scale[0].base,
288 st->scale[0].base + st->scale[0].length - 1);
289 if (st->scale[0].region < 0 || st->scale[0].base == U64_MAX ||
290 st->scale[0].length == U32_MAX)
291 return -EINVAL;
292 info->region_size[st->scale[0].region] =
293 max(info->region_size[st->scale[0].region],
294 st->scale[0].base + st->scale[0].length);
295 }
296
297 len = feat_matrix_length(info, &st->ofm, st->ofm.width,
298 st->ofm.height[2], st->ofm.depth);
299 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n",
300 op, st->ofm.region, st->ofm.base[0], len);
301 if (len == U64_MAX)
302 return -EINVAL;
303 info->output_region[st->ofm.region] = true;
304
305 return 0;
306 }
307
calc_sizes_elemwise(struct drm_device * ddev,struct ethosu_validated_cmdstream_info * info,u16 op,struct cmd_state * st,bool ifm,bool ifm2)308 static int calc_sizes_elemwise(struct drm_device *ddev,
309 struct ethosu_validated_cmdstream_info *info,
310 u16 op, struct cmd_state *st,
311 bool ifm, bool ifm2)
312 {
313 u32 height, width, depth;
314 u64 len;
315
316 if (ifm) {
317 height = st->ifm.broadcast & 0x1 ? 0 : st->ofm.height[2];
318 width = st->ifm.broadcast & 0x2 ? 0 : st->ofm.width;
319 depth = st->ifm.broadcast & 0x4 ? 0 : st->ofm.depth;
320
321 len = feat_matrix_length(info, &st->ifm, width,
322 height, depth);
323 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n",
324 op, st->ifm.region, st->ifm.base[0], len);
325 if (len == U64_MAX)
326 return -EINVAL;
327 }
328
329 if (ifm2) {
330 height = st->ifm2.broadcast & 0x1 ? 0 : st->ofm.height[2];
331 width = st->ifm2.broadcast & 0x2 ? 0 : st->ofm.width;
332 depth = st->ifm2.broadcast & 0x4 ? 0 : st->ofm.depth;
333
334 len = feat_matrix_length(info, &st->ifm2, width,
335 height, depth);
336 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n",
337 op, st->ifm2.region, st->ifm2.base[0], len);
338 if (len == U64_MAX)
339 return -EINVAL;
340 }
341
342 len = feat_matrix_length(info, &st->ofm, st->ofm.width,
343 st->ofm.height[2], st->ofm.depth);
344 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n",
345 op, st->ofm.region, st->ofm.base[0], len);
346 if (len == U64_MAX)
347 return -EINVAL;
348 info->output_region[st->ofm.region] = true;
349
350 return 0;
351 }
352
ethosu_gem_cmdstream_copy_and_validate(struct drm_device * ddev,u32 __user * ucmds,struct ethosu_gem_object * bo,u32 size)353 static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev,
354 u32 __user *ucmds,
355 struct ethosu_gem_object *bo,
356 u32 size)
357 {
358 struct ethosu_validated_cmdstream_info __free(kfree) *info = kzalloc_obj(*info);
359 struct ethosu_device *edev = to_ethosu_device(ddev);
360 u32 *bocmds = bo->base.vaddr;
361 struct cmd_state st;
362 int i, ret;
363
364 if (!info)
365 return -ENOMEM;
366 info->cmd_size = size;
367
368 cmd_state_init(&st);
369
370 for (i = 0; i < size / 4; i++) {
371 bool use_ifm, use_ifm2, use_scale;
372 u64 dstlen, srclen;
373 u16 cmd, param;
374 u32 cmds[2];
375 u64 addr;
376
377 if (get_user(cmds[0], ucmds++))
378 return -EFAULT;
379
380 bocmds[i] = cmds[0];
381
382 cmd = cmds[0];
383 param = cmds[0] >> 16;
384
385 if (cmd & 0x4000) {
386 if (get_user(cmds[1], ucmds++))
387 return -EFAULT;
388
389 i++;
390 bocmds[i] = cmds[1];
391 addr = cmd_to_addr(cmds);
392 }
393
394 switch (cmd) {
395 case NPU_OP_DMA_START:
396 srclen = dma_length(info, &st.dma, &st.dma.src);
397 dstlen = dma_length(info, &st.dma, &st.dma.dst);
398
399 if (st.dma.dst.region >= 0)
400 info->output_region[st.dma.dst.region] = true;
401 dev_dbg(ddev->dev, "cmd: DMA SRC:%d:0x%llx+0x%llx DST:%d:0x%llx+0x%llx\n",
402 st.dma.src.region, st.dma.src.offset, srclen,
403 st.dma.dst.region, st.dma.dst.offset, dstlen);
404 break;
405 case NPU_OP_CONV:
406 case NPU_OP_DEPTHWISE:
407 use_ifm2 = param & 0x1; // weights_ifm2
408 use_scale = !(st.ofm.precision & 0x100);
409 ret = calc_sizes(ddev, info, cmd, &st, true, use_ifm2,
410 !use_ifm2, use_scale);
411 if (ret)
412 return ret;
413 break;
414 case NPU_OP_POOL:
415 use_ifm = param != 0x4; // pooling mode
416 use_scale = !(st.ofm.precision & 0x100);
417 ret = calc_sizes(ddev, info, cmd, &st, use_ifm, false,
418 false, use_scale);
419 if (ret)
420 return ret;
421 break;
422 case NPU_OP_ELEMENTWISE:
423 use_scale = ethosu_is_u65(edev) ?
424 (st.ifm2.broadcast & 0x80) :
425 (st.ifm2.broadcast == 8);
426 use_ifm2 = !(use_scale || (param == 5) ||
427 (param == 6) || (param == 7) || (param == 0x24));
428 use_ifm = st.ifm.broadcast != 8;
429 ret = calc_sizes_elemwise(ddev, info, cmd, &st, use_ifm, use_ifm2);
430 if (ret)
431 return ret;
432 break;
433 case NPU_OP_RESIZE: // U85 only
434 WARN_ON(1); // TODO
435 break;
436 case NPU_SET_KERNEL_WIDTH_M1:
437 st.ifm.width = param;
438 break;
439 case NPU_SET_KERNEL_HEIGHT_M1:
440 st.ifm.height[2] = param;
441 break;
442 case NPU_SET_KERNEL_STRIDE:
443 st.ifm.stride_kernel = param;
444 break;
445 case NPU_SET_IFM_PAD_TOP:
446 st.ifm.pad_top = param & 0x7f;
447 break;
448 case NPU_SET_IFM_PAD_LEFT:
449 st.ifm.pad_left = param & 0x7f;
450 break;
451 case NPU_SET_IFM_PAD_RIGHT:
452 st.ifm.pad_right = param & 0xff;
453 break;
454 case NPU_SET_IFM_PAD_BOTTOM:
455 st.ifm.pad_bottom = param & 0xff;
456 break;
457 case NPU_SET_IFM_DEPTH_M1:
458 st.ifm.depth = param;
459 break;
460 case NPU_SET_IFM_PRECISION:
461 st.ifm.precision = param;
462 break;
463 case NPU_SET_IFM_BROADCAST:
464 st.ifm.broadcast = param;
465 break;
466 case NPU_SET_IFM_REGION:
467 st.ifm.region = param & 0x7f;
468 break;
469 case NPU_SET_IFM_WIDTH0_M1:
470 st.ifm.width0 = param;
471 break;
472 case NPU_SET_IFM_HEIGHT0_M1:
473 st.ifm.height[0] = param;
474 break;
475 case NPU_SET_IFM_HEIGHT1_M1:
476 st.ifm.height[1] = param;
477 break;
478 case NPU_SET_IFM_BASE0:
479 case NPU_SET_IFM_BASE1:
480 case NPU_SET_IFM_BASE2:
481 case NPU_SET_IFM_BASE3:
482 st.ifm.base[cmd & 0x3] = addr;
483 break;
484 case NPU_SET_IFM_STRIDE_X:
485 st.ifm.stride_x = addr;
486 break;
487 case NPU_SET_IFM_STRIDE_Y:
488 st.ifm.stride_y = addr;
489 break;
490 case NPU_SET_IFM_STRIDE_C:
491 st.ifm.stride_c = addr;
492 break;
493
494 case NPU_SET_OFM_WIDTH_M1:
495 st.ofm.width = param;
496 break;
497 case NPU_SET_OFM_HEIGHT_M1:
498 st.ofm.height[2] = param;
499 break;
500 case NPU_SET_OFM_DEPTH_M1:
501 st.ofm.depth = param;
502 break;
503 case NPU_SET_OFM_PRECISION:
504 st.ofm.precision = param;
505 break;
506 case NPU_SET_OFM_REGION:
507 st.ofm.region = param & 0x7;
508 break;
509 case NPU_SET_OFM_WIDTH0_M1:
510 st.ofm.width0 = param;
511 break;
512 case NPU_SET_OFM_HEIGHT0_M1:
513 st.ofm.height[0] = param;
514 break;
515 case NPU_SET_OFM_HEIGHT1_M1:
516 st.ofm.height[1] = param;
517 break;
518 case NPU_SET_OFM_BASE0:
519 case NPU_SET_OFM_BASE1:
520 case NPU_SET_OFM_BASE2:
521 case NPU_SET_OFM_BASE3:
522 st.ofm.base[cmd & 0x3] = addr;
523 break;
524 case NPU_SET_OFM_STRIDE_X:
525 st.ofm.stride_x = addr;
526 break;
527 case NPU_SET_OFM_STRIDE_Y:
528 st.ofm.stride_y = addr;
529 break;
530 case NPU_SET_OFM_STRIDE_C:
531 st.ofm.stride_c = addr;
532 break;
533
534 case NPU_SET_IFM2_BROADCAST:
535 st.ifm2.broadcast = param;
536 break;
537 case NPU_SET_IFM2_PRECISION:
538 st.ifm2.precision = param;
539 break;
540 case NPU_SET_IFM2_REGION:
541 st.ifm2.region = param & 0x7;
542 break;
543 case NPU_SET_IFM2_WIDTH0_M1:
544 st.ifm2.width0 = param;
545 break;
546 case NPU_SET_IFM2_HEIGHT0_M1:
547 st.ifm2.height[0] = param;
548 break;
549 case NPU_SET_IFM2_HEIGHT1_M1:
550 st.ifm2.height[1] = param;
551 break;
552 case NPU_SET_IFM2_BASE0:
553 case NPU_SET_IFM2_BASE1:
554 case NPU_SET_IFM2_BASE2:
555 case NPU_SET_IFM2_BASE3:
556 st.ifm2.base[cmd & 0x3] = addr;
557 break;
558 case NPU_SET_IFM2_STRIDE_X:
559 st.ifm2.stride_x = addr;
560 break;
561 case NPU_SET_IFM2_STRIDE_Y:
562 st.ifm2.stride_y = addr;
563 break;
564 case NPU_SET_IFM2_STRIDE_C:
565 st.ifm2.stride_c = addr;
566 break;
567
568 case NPU_SET_WEIGHT_REGION:
569 st.weight[0].region = param & 0x7;
570 break;
571 case NPU_SET_SCALE_REGION:
572 st.scale[0].region = param & 0x7;
573 break;
574 case NPU_SET_WEIGHT_BASE:
575 st.weight[0].base = addr;
576 break;
577 case NPU_SET_WEIGHT_LENGTH:
578 st.weight[0].length = cmds[1];
579 break;
580 case NPU_SET_SCALE_BASE:
581 st.scale[0].base = addr;
582 break;
583 case NPU_SET_SCALE_LENGTH:
584 st.scale[0].length = cmds[1];
585 break;
586 case NPU_SET_WEIGHT1_BASE:
587 st.weight[1].base = addr;
588 break;
589 case NPU_SET_WEIGHT1_LENGTH:
590 st.weight[1].length = cmds[1];
591 break;
592 case NPU_SET_SCALE1_BASE: // NPU_SET_WEIGHT2_BASE (U85)
593 if (ethosu_is_u65(edev))
594 st.scale[1].base = addr;
595 else
596 st.weight[2].base = addr;
597 break;
598 case NPU_SET_SCALE1_LENGTH: // NPU_SET_WEIGHT2_LENGTH (U85)
599 if (ethosu_is_u65(edev))
600 st.scale[1].length = cmds[1];
601 else
602 st.weight[1].length = cmds[1];
603 break;
604 case NPU_SET_WEIGHT3_BASE:
605 st.weight[3].base = addr;
606 break;
607 case NPU_SET_WEIGHT3_LENGTH:
608 st.weight[3].length = cmds[1];
609 break;
610
611 case NPU_SET_DMA0_SRC_REGION:
612 if (param & 0x100)
613 st.dma.src.region = -1;
614 else
615 st.dma.src.region = param & 0x7;
616 st.dma.mode = (param >> 9) & 0x3;
617 break;
618 case NPU_SET_DMA0_DST_REGION:
619 if (param & 0x100)
620 st.dma.dst.region = -1;
621 else
622 st.dma.dst.region = param & 0x7;
623 break;
624 case NPU_SET_DMA0_SIZE0:
625 st.dma.size0 = param;
626 break;
627 case NPU_SET_DMA0_SIZE1:
628 st.dma.size1 = param;
629 break;
630 case NPU_SET_DMA0_SRC_STRIDE0:
631 st.dma.src.stride[0] = ((s64)addr << 24) >> 24;
632 break;
633 case NPU_SET_DMA0_SRC_STRIDE1:
634 st.dma.src.stride[1] = ((s64)addr << 24) >> 24;
635 break;
636 case NPU_SET_DMA0_DST_STRIDE0:
637 st.dma.dst.stride[0] = ((s64)addr << 24) >> 24;
638 break;
639 case NPU_SET_DMA0_DST_STRIDE1:
640 st.dma.dst.stride[1] = ((s64)addr << 24) >> 24;
641 break;
642 case NPU_SET_DMA0_SRC:
643 st.dma.src.offset = addr;
644 break;
645 case NPU_SET_DMA0_DST:
646 st.dma.dst.offset = addr;
647 break;
648 case NPU_SET_DMA0_LEN:
649 st.dma.src.len = st.dma.dst.len = addr;
650 break;
651 default:
652 break;
653 }
654 }
655
656 for (i = 0; i < NPU_BASEP_REGION_MAX; i++) {
657 if (!info->region_size[i])
658 continue;
659 dev_dbg(ddev->dev, "region %d max size: 0x%llx\n",
660 i, info->region_size[i]);
661 }
662
663 bo->info = no_free_ptr(info);
664 return 0;
665 }
666
667 /**
668 * ethosu_gem_cmdstream_create() - Create a GEM object and attach it to a handle.
669 * @file: DRM file.
670 * @ddev: DRM device.
671 * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared.
672 * @size: Size of the GEM object to allocate.
673 * @flags: Combination of drm_ethosu_bo_flags flags.
674 * @handle: Pointer holding the handle pointing to the new GEM object.
675 *
676 * Return: Zero on success
677 */
ethosu_gem_cmdstream_create(struct drm_file * file,struct drm_device * ddev,u32 size,u64 data,u32 flags,u32 * handle)678 int ethosu_gem_cmdstream_create(struct drm_file *file,
679 struct drm_device *ddev,
680 u32 size, u64 data, u32 flags, u32 *handle)
681 {
682 int ret;
683 struct drm_gem_dma_object *mem;
684 struct ethosu_gem_object *bo;
685
686 mem = drm_gem_dma_create(ddev, size);
687 if (IS_ERR(mem))
688 return PTR_ERR(mem);
689
690 bo = to_ethosu_bo(&mem->base);
691 bo->flags = flags;
692
693 ret = ethosu_gem_cmdstream_copy_and_validate(ddev,
694 (void __user *)(uintptr_t)data,
695 bo, size);
696 if (ret)
697 goto fail;
698
699 /*
700 * Allocate an id of idr table where the obj is registered
701 * and handle has the id what user can see.
702 */
703 ret = drm_gem_handle_create(file, &mem->base, handle);
704
705 fail:
706 /* drop reference from allocate - handle holds it now. */
707 drm_gem_object_put(&mem->base);
708
709 return ret;
710 }
711