1 // SPDX-License-Identifier: GPL-2.0-only or MIT
2 /* Copyright 2025 Arm, Ltd. */
3
4 #include <linux/err.h>
5 #include <linux/overflow.h>
6 #include <linux/slab.h>
7
8 #include <drm/ethosu_accel.h>
9
10 #include "ethosu_device.h"
11 #include "ethosu_gem.h"
12
ethosu_gem_free_object(struct drm_gem_object * obj)13 static void ethosu_gem_free_object(struct drm_gem_object *obj)
14 {
15 struct ethosu_gem_object *bo = to_ethosu_bo(obj);
16
17 kfree(bo->info);
18 drm_gem_free_mmap_offset(&bo->base.base);
19 drm_gem_dma_free(&bo->base);
20 }
21
ethosu_gem_mmap(struct drm_gem_object * obj,struct vm_area_struct * vma)22 static int ethosu_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
23 {
24 struct ethosu_gem_object *bo = to_ethosu_bo(obj);
25
26 /* Don't allow mmap on objects that have the NO_MMAP flag set. */
27 if (bo->flags & DRM_ETHOSU_BO_NO_MMAP)
28 return -EINVAL;
29
30 return drm_gem_dma_object_mmap(obj, vma);
31 }
32
33 static const struct drm_gem_object_funcs ethosu_gem_funcs = {
34 .free = ethosu_gem_free_object,
35 .print_info = drm_gem_dma_object_print_info,
36 .get_sg_table = drm_gem_dma_object_get_sg_table,
37 .vmap = drm_gem_dma_object_vmap,
38 .mmap = ethosu_gem_mmap,
39 .vm_ops = &drm_gem_dma_vm_ops,
40 };
41
42 /**
43 * ethosu_gem_create_object - Implementation of driver->gem_create_object.
44 * @ddev: DRM device
45 * @size: Size in bytes of the memory the object will reference
46 *
47 * This lets the GEM helpers allocate object structs for us, and keep
48 * our BO stats correct.
49 */
ethosu_gem_create_object(struct drm_device * ddev,size_t size)50 struct drm_gem_object *ethosu_gem_create_object(struct drm_device *ddev, size_t size)
51 {
52 struct ethosu_gem_object *obj;
53
54 obj = kzalloc_obj(*obj);
55 if (!obj)
56 return ERR_PTR(-ENOMEM);
57
58 obj->base.base.funcs = ðosu_gem_funcs;
59 return &obj->base.base;
60 }
61
62 /**
63 * ethosu_gem_create_with_handle() - Create a GEM object and attach it to a handle.
64 * @file: DRM file.
65 * @ddev: DRM device.
66 * @size: Size of the GEM object to allocate.
67 * @flags: Combination of drm_ethosu_bo_flags flags.
68 * @handle: Pointer holding the handle pointing to the new GEM object.
69 *
70 * Return: Zero on success
71 */
ethosu_gem_create_with_handle(struct drm_file * file,struct drm_device * ddev,u64 * size,u32 flags,u32 * handle)72 int ethosu_gem_create_with_handle(struct drm_file *file,
73 struct drm_device *ddev,
74 u64 *size, u32 flags, u32 *handle)
75 {
76 struct drm_gem_dma_object *mem;
77 struct ethosu_gem_object *bo;
78 int ret;
79
80 mem = drm_gem_dma_create(ddev, *size);
81 if (IS_ERR(mem))
82 return PTR_ERR(mem);
83
84 bo = to_ethosu_bo(&mem->base);
85 bo->flags = flags;
86
87 /*
88 * Allocate an id of idr table where the obj is registered
89 * and handle has the id what user can see.
90 */
91 ret = drm_gem_handle_create(file, &mem->base, handle);
92 if (!ret)
93 *size = bo->base.base.size;
94
95 /* drop reference from allocate - handle holds it now. */
96 drm_gem_object_put(&mem->base);
97
98 return ret;
99 }
100
101 struct dma {
102 s8 region;
103 u64 len;
104 u64 offset;
105 s64 stride[2];
106 };
107
108 struct dma_state {
109 u16 size0;
110 u16 size1;
111 s8 mode;
112 struct dma src;
113 struct dma dst;
114 };
115
116 struct buffer {
117 u64 base;
118 u32 length;
119 s8 region;
120 };
121
122 struct feat_matrix {
123 u64 base[4];
124 s64 stride_x;
125 s64 stride_y;
126 s64 stride_c;
127 s8 region;
128 u8 broadcast;
129 u16 stride_kernel;
130 u16 precision;
131 u16 depth;
132 u16 width;
133 u16 width0;
134 u16 height[3];
135 u8 pad_top;
136 u8 pad_left;
137 u8 pad_bottom;
138 u8 pad_right;
139 };
140
141 struct cmd_state {
142 struct dma_state dma;
143 struct buffer scale[2];
144 struct buffer weight[4];
145 struct feat_matrix ofm;
146 struct feat_matrix ifm;
147 struct feat_matrix ifm2;
148 };
149
cmd_state_init(struct cmd_state * st)150 static void cmd_state_init(struct cmd_state *st)
151 {
152 /* Initialize to all 1s to detect missing setup */
153 memset(st, 0xff, sizeof(*st));
154 }
155
cmd_to_addr(u32 * cmd)156 static u64 cmd_to_addr(u32 *cmd)
157 {
158 return (((u64)cmd[0] & 0xff0000) << 16) | cmd[1];
159 }
160
dma_length(struct ethosu_validated_cmdstream_info * info,struct dma_state * dma_st,struct dma * dma)161 static u64 dma_length(struct ethosu_validated_cmdstream_info *info,
162 struct dma_state *dma_st, struct dma *dma)
163 {
164 s8 mode = dma_st->mode;
165 u64 len = dma->len;
166
167 if (len == U64_MAX)
168 return U64_MAX;
169
170 if (mode >= 1) {
171 if (dma->stride[0] < 0 && (u64)(-dma->stride[0]) > len)
172 return U64_MAX;
173 len += dma->stride[0];
174 if (check_mul_overflow(len, (u64)dma_st->size0, &len))
175 return U64_MAX;
176 }
177 if (mode == 2) {
178 if (dma->stride[1] < 0 && (u64)(-dma->stride[1]) > len)
179 return U64_MAX;
180 len += dma->stride[1];
181 if (check_mul_overflow(len, (u64)dma_st->size1, &len))
182 return U64_MAX;
183 }
184 if (dma->region >= 0) {
185 u64 end;
186
187 if (check_add_overflow(len, dma->offset, &end))
188 return U64_MAX;
189 info->region_size[dma->region] = max(info->region_size[dma->region], end);
190 }
191
192 return len;
193 }
194
feat_matrix_chained(struct ethosu_device * edev,struct feat_matrix * fm)195 static bool feat_matrix_chained(struct ethosu_device *edev, struct feat_matrix *fm)
196 {
197 u32 storage = fm->precision >> 14;
198
199 return !ethosu_is_u65(edev) && storage == 2;
200 }
201
feat_matrix_length(struct ethosu_device * edev,struct ethosu_validated_cmdstream_info * info,struct feat_matrix * fm,u32 x,u32 y,u32 c,bool ofm)202 static u64 feat_matrix_length(struct ethosu_device *edev,
203 struct ethosu_validated_cmdstream_info *info,
204 struct feat_matrix *fm,
205 u32 x, u32 y, u32 c, bool ofm)
206 {
207 u32 element_size, storage = ethosu_is_u65(edev) ? 0 : fm->precision >> 14;
208 int tile = 0;
209 u64 addr;
210
211 if (fm->region < 0)
212 return U64_MAX;
213
214 if (feat_matrix_chained(edev, fm))
215 return 0;
216
217 switch (storage) {
218 case 0:
219 if (x >= fm->width0 + 1) {
220 x -= fm->width0 + 1;
221 tile += 1;
222 }
223 if (y >= fm->height[tile] + 1) {
224 y -= fm->height[tile] + 1;
225 tile += 2;
226 }
227 break;
228 case 1:
229 if (y >= fm->height[1] + 1) {
230 y -= fm->height[1] + 1;
231 tile = 2;
232 } else if (y >= fm->height[0] + 1) {
233 y -= fm->height[0] + 1;
234 tile = 1;
235 }
236 break;
237 default:
238 return U64_MAX;
239 }
240 if (fm->base[tile] == U64_MAX)
241 return U64_MAX;
242
243 addr = fm->base[tile] + y * fm->stride_y;
244
245 switch ((fm->precision >> 6) & 0x3) { // format
246 case 0: //nhwc:
247 element_size = BIT((fm->precision >> (ofm ? 1 : 2)) & 0x3);
248 addr += x * fm->stride_x + c * element_size;
249 break;
250 case 1: //nhcwb16:
251 element_size = BIT((fm->precision >> (ofm ? 1 : 2)) & 0x3);
252
253 addr += (c / 16) * fm->stride_c + (16 * x + (c & 0xf)) * element_size;
254 break;
255 }
256
257 info->region_size[fm->region] = max(info->region_size[fm->region], addr + 1);
258
259 return addr;
260 }
261
calc_sizes(struct drm_device * ddev,struct ethosu_validated_cmdstream_info * info,u16 op,struct cmd_state * st,bool ifm,bool ifm2,bool weight,bool scale)262 static int calc_sizes(struct drm_device *ddev,
263 struct ethosu_validated_cmdstream_info *info,
264 u16 op, struct cmd_state *st,
265 bool ifm, bool ifm2, bool weight, bool scale)
266 {
267 struct ethosu_device *edev = to_ethosu_device(ddev);
268 u64 len;
269
270 if (ifm) {
271 if (st->ifm.stride_kernel == U16_MAX)
272 return -EINVAL;
273 u32 stride_y = ((st->ifm.stride_kernel >> 8) & 0x2) +
274 ((st->ifm.stride_kernel >> 1) & 0x1) + 1;
275 u32 stride_x = ((st->ifm.stride_kernel >> 5) & 0x2) +
276 (st->ifm.stride_kernel & 0x1) + 1;
277 s32 ifm_height = st->ofm.height[2] * stride_y +
278 st->ifm.height[2] - (st->ifm.pad_top + st->ifm.pad_bottom);
279 s32 ifm_width = st->ofm.width * stride_x +
280 st->ifm.width - (st->ifm.pad_left + st->ifm.pad_right);
281
282 if (ifm_height < 0 || ifm_width < 0)
283 return -EINVAL;
284
285 len = feat_matrix_length(edev, info, &st->ifm, ifm_width,
286 ifm_height, st->ifm.depth, false);
287 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n",
288 op, st->ifm.region, st->ifm.base[0], len);
289 if (len == U64_MAX)
290 return -EINVAL;
291 }
292
293 if (ifm2) {
294 len = feat_matrix_length(edev, info, &st->ifm2, st->ifm.depth,
295 0, st->ofm.depth, false);
296 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n",
297 op, st->ifm2.region, st->ifm2.base[0], len);
298 if (len == U64_MAX)
299 return -EINVAL;
300 }
301
302 if (weight) {
303 dev_dbg(ddev->dev, "op %d: W:%d:0x%llx-0x%llx\n",
304 op, st->weight[0].region, st->weight[0].base,
305 st->weight[0].base + st->weight[0].length - 1);
306 if (st->weight[0].region < 0 || st->weight[0].base == U64_MAX ||
307 st->weight[0].length == U32_MAX)
308 return -EINVAL;
309 info->region_size[st->weight[0].region] =
310 max(info->region_size[st->weight[0].region],
311 st->weight[0].base + st->weight[0].length);
312 }
313
314 if (scale) {
315 dev_dbg(ddev->dev, "op %d: S:%d:0x%llx-0x%llx\n",
316 op, st->scale[0].region, st->scale[0].base,
317 st->scale[0].base + st->scale[0].length - 1);
318 if (st->scale[0].region < 0 || st->scale[0].base == U64_MAX ||
319 st->scale[0].length == U32_MAX)
320 return -EINVAL;
321 info->region_size[st->scale[0].region] =
322 max(info->region_size[st->scale[0].region],
323 st->scale[0].base + st->scale[0].length);
324 }
325
326 len = feat_matrix_length(edev, info, &st->ofm, st->ofm.width,
327 st->ofm.height[2], st->ofm.depth, true);
328 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n",
329 op, st->ofm.region, st->ofm.base[0], len);
330 if (len == U64_MAX)
331 return -EINVAL;
332 if (!feat_matrix_chained(edev, &st->ofm))
333 info->output_region[st->ofm.region] = true;
334
335 return 0;
336 }
337
calc_sizes_elemwise(struct drm_device * ddev,struct ethosu_validated_cmdstream_info * info,u16 op,struct cmd_state * st,bool ifm,bool ifm2)338 static int calc_sizes_elemwise(struct drm_device *ddev,
339 struct ethosu_validated_cmdstream_info *info,
340 u16 op, struct cmd_state *st,
341 bool ifm, bool ifm2)
342 {
343 struct ethosu_device *edev = to_ethosu_device(ddev);
344 u32 height, width, depth;
345 u64 len;
346
347 if (ifm) {
348 height = st->ifm.broadcast & 0x1 ? 0 : st->ofm.height[2];
349 width = st->ifm.broadcast & 0x2 ? 0 : st->ofm.width;
350 depth = st->ifm.broadcast & 0x4 ? 0 : st->ofm.depth;
351
352 len = feat_matrix_length(edev, info, &st->ifm, width,
353 height, depth, false);
354 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n",
355 op, st->ifm.region, st->ifm.base[0], len);
356 if (len == U64_MAX)
357 return -EINVAL;
358 }
359
360 if (ifm2) {
361 height = st->ifm2.broadcast & 0x1 ? 0 : st->ofm.height[2];
362 width = st->ifm2.broadcast & 0x2 ? 0 : st->ofm.width;
363 depth = st->ifm2.broadcast & 0x4 ? 0 : st->ofm.depth;
364
365 len = feat_matrix_length(edev, info, &st->ifm2, width,
366 height, depth, false);
367 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n",
368 op, st->ifm2.region, st->ifm2.base[0], len);
369 if (len == U64_MAX)
370 return -EINVAL;
371 }
372
373 len = feat_matrix_length(edev, info, &st->ofm, st->ofm.width,
374 st->ofm.height[2], st->ofm.depth, true);
375 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n",
376 op, st->ofm.region, st->ofm.base[0], len);
377 if (len == U64_MAX)
378 return -EINVAL;
379 if (!feat_matrix_chained(edev, &st->ofm))
380 info->output_region[st->ofm.region] = true;
381
382 return 0;
383 }
384
ethosu_gem_cmdstream_copy_and_validate(struct drm_device * ddev,u32 __user * ucmds,struct ethosu_gem_object * bo,u32 size)385 static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev,
386 u32 __user *ucmds,
387 struct ethosu_gem_object *bo,
388 u32 size)
389 {
390 struct ethosu_validated_cmdstream_info __free(kfree) *info = kzalloc_obj(*info);
391 struct ethosu_device *edev = to_ethosu_device(ddev);
392 u32 *bocmds = bo->base.vaddr;
393 bool ends_with_stop = false;
394 struct cmd_state st;
395 int i, ret;
396
397 if (!info)
398 return -ENOMEM;
399 info->cmd_size = size;
400
401 cmd_state_init(&st);
402
403 for (i = 0; i < size / 4; i++) {
404 bool use_ifm, use_ifm2, use_scale;
405 u64 dstlen, srclen;
406 u16 cmd, param;
407 u32 cmds[2];
408 u64 addr;
409
410 if (get_user(cmds[0], ucmds++))
411 return -EFAULT;
412
413 bocmds[i] = cmds[0];
414
415 cmd = cmds[0];
416 param = cmds[0] >> 16;
417
418 if (cmd & 0x4000) {
419 if (get_user(cmds[1], ucmds++))
420 return -EFAULT;
421
422 i++;
423 if (i >= size / 4)
424 return -EINVAL;
425 bocmds[i] = cmds[1];
426 addr = cmd_to_addr(cmds);
427 }
428
429 switch (cmd) {
430 case NPU_OP_STOP:
431 if (i != size / 4 - 1)
432 return -EINVAL;
433 ends_with_stop = true;
434 break;
435 case NPU_OP_DMA_START:
436 srclen = dma_length(info, &st.dma, &st.dma.src);
437 dstlen = dma_length(info, &st.dma, &st.dma.dst);
438 if (srclen == U64_MAX || dstlen == U64_MAX)
439 return -EINVAL;
440
441 if (st.dma.dst.region >= 0)
442 info->output_region[st.dma.dst.region] = true;
443 dev_dbg(ddev->dev, "cmd: DMA SRC:%d:0x%llx+0x%llx DST:%d:0x%llx+0x%llx\n",
444 st.dma.src.region, st.dma.src.offset, srclen,
445 st.dma.dst.region, st.dma.dst.offset, dstlen);
446 break;
447 case NPU_OP_CONV:
448 case NPU_OP_DEPTHWISE:
449 use_ifm2 = param & 0x1; // weights_ifm2
450 use_scale = !(st.ofm.precision & 0x100);
451 ret = calc_sizes(ddev, info, cmd, &st, true, use_ifm2,
452 !use_ifm2, use_scale);
453 if (ret)
454 return ret;
455 break;
456 case NPU_OP_POOL:
457 use_ifm = param != 0x4; // pooling mode
458 use_scale = !(st.ofm.precision & 0x100);
459 ret = calc_sizes(ddev, info, cmd, &st, use_ifm, false,
460 false, use_scale);
461 if (ret)
462 return ret;
463 break;
464 case NPU_OP_ELEMENTWISE:
465 use_scale = ethosu_is_u65(edev) ?
466 (st.ifm2.broadcast & 0x80) :
467 (st.ifm2.broadcast == 8);
468 use_ifm2 = !(use_scale || (param == 5) ||
469 (param == 6) || (param == 7) || (param == 0x24));
470 use_ifm = st.ifm.broadcast != 8;
471 ret = calc_sizes_elemwise(ddev, info, cmd, &st, use_ifm, use_ifm2);
472 if (ret)
473 return ret;
474 break;
475 case NPU_OP_RESIZE: // U85 only
476 return -EINVAL;
477 case NPU_SET_KERNEL_WIDTH_M1:
478 st.ifm.width = param;
479 break;
480 case NPU_SET_KERNEL_HEIGHT_M1:
481 st.ifm.height[2] = param;
482 break;
483 case NPU_SET_KERNEL_STRIDE:
484 st.ifm.stride_kernel = param;
485 break;
486 case NPU_SET_IFM_PAD_TOP:
487 st.ifm.pad_top = param & 0x7f;
488 break;
489 case NPU_SET_IFM_PAD_LEFT:
490 st.ifm.pad_left = param & 0x7f;
491 break;
492 case NPU_SET_IFM_PAD_RIGHT:
493 st.ifm.pad_right = param & 0xff;
494 break;
495 case NPU_SET_IFM_PAD_BOTTOM:
496 st.ifm.pad_bottom = param & 0xff;
497 break;
498 case NPU_SET_IFM_DEPTH_M1:
499 st.ifm.depth = param;
500 break;
501 case NPU_SET_IFM_PRECISION:
502 st.ifm.precision = param;
503 break;
504 case NPU_SET_IFM_BROADCAST:
505 st.ifm.broadcast = param;
506 break;
507 case NPU_SET_IFM_REGION:
508 st.ifm.region = param & 0x7;
509 break;
510 case NPU_SET_IFM_WIDTH0_M1:
511 st.ifm.width0 = param;
512 break;
513 case NPU_SET_IFM_HEIGHT0_M1:
514 st.ifm.height[0] = param;
515 break;
516 case NPU_SET_IFM_HEIGHT1_M1:
517 st.ifm.height[1] = param;
518 break;
519 case NPU_SET_IFM_BASE0:
520 case NPU_SET_IFM_BASE1:
521 case NPU_SET_IFM_BASE2:
522 case NPU_SET_IFM_BASE3:
523 st.ifm.base[cmd & 0x3] = addr;
524 break;
525 case NPU_SET_IFM_STRIDE_X:
526 st.ifm.stride_x = addr;
527 break;
528 case NPU_SET_IFM_STRIDE_Y:
529 st.ifm.stride_y = addr;
530 break;
531 case NPU_SET_IFM_STRIDE_C:
532 st.ifm.stride_c = addr;
533 break;
534
535 case NPU_SET_OFM_WIDTH_M1:
536 st.ofm.width = param;
537 break;
538 case NPU_SET_OFM_HEIGHT_M1:
539 st.ofm.height[2] = param;
540 break;
541 case NPU_SET_OFM_DEPTH_M1:
542 st.ofm.depth = param;
543 break;
544 case NPU_SET_OFM_PRECISION:
545 st.ofm.precision = param;
546 break;
547 case NPU_SET_OFM_REGION:
548 st.ofm.region = param & 0x7;
549 break;
550 case NPU_SET_OFM_WIDTH0_M1:
551 st.ofm.width0 = param;
552 break;
553 case NPU_SET_OFM_HEIGHT0_M1:
554 st.ofm.height[0] = param;
555 break;
556 case NPU_SET_OFM_HEIGHT1_M1:
557 st.ofm.height[1] = param;
558 break;
559 case NPU_SET_OFM_BASE0:
560 case NPU_SET_OFM_BASE1:
561 case NPU_SET_OFM_BASE2:
562 case NPU_SET_OFM_BASE3:
563 st.ofm.base[cmd & 0x3] = addr;
564 break;
565 case NPU_SET_OFM_STRIDE_X:
566 st.ofm.stride_x = addr;
567 break;
568 case NPU_SET_OFM_STRIDE_Y:
569 st.ofm.stride_y = addr;
570 break;
571 case NPU_SET_OFM_STRIDE_C:
572 st.ofm.stride_c = addr;
573 break;
574
575 case NPU_SET_IFM2_BROADCAST:
576 st.ifm2.broadcast = param;
577 break;
578 case NPU_SET_IFM2_PRECISION:
579 st.ifm2.precision = param;
580 break;
581 case NPU_SET_IFM2_REGION:
582 st.ifm2.region = param & 0x7;
583 break;
584 case NPU_SET_IFM2_WIDTH0_M1:
585 st.ifm2.width0 = param;
586 break;
587 case NPU_SET_IFM2_HEIGHT0_M1:
588 st.ifm2.height[0] = param;
589 break;
590 case NPU_SET_IFM2_HEIGHT1_M1:
591 st.ifm2.height[1] = param;
592 break;
593 case NPU_SET_IFM2_BASE0:
594 case NPU_SET_IFM2_BASE1:
595 case NPU_SET_IFM2_BASE2:
596 case NPU_SET_IFM2_BASE3:
597 st.ifm2.base[cmd & 0x3] = addr;
598 break;
599 case NPU_SET_IFM2_STRIDE_X:
600 st.ifm2.stride_x = addr;
601 break;
602 case NPU_SET_IFM2_STRIDE_Y:
603 st.ifm2.stride_y = addr;
604 break;
605 case NPU_SET_IFM2_STRIDE_C:
606 st.ifm2.stride_c = addr;
607 break;
608
609 case NPU_SET_WEIGHT_REGION:
610 st.weight[0].region = param & 0x7;
611 break;
612 case NPU_SET_SCALE_REGION:
613 st.scale[0].region = param & 0x7;
614 break;
615 case NPU_SET_WEIGHT_BASE:
616 st.weight[0].base = addr;
617 break;
618 case NPU_SET_WEIGHT_LENGTH:
619 st.weight[0].length = cmds[1];
620 break;
621 case NPU_SET_SCALE_BASE:
622 st.scale[0].base = addr;
623 break;
624 case NPU_SET_SCALE_LENGTH:
625 st.scale[0].length = cmds[1];
626 break;
627 case NPU_SET_WEIGHT1_BASE:
628 st.weight[1].base = addr;
629 break;
630 case NPU_SET_WEIGHT1_LENGTH:
631 st.weight[1].length = cmds[1];
632 break;
633 case NPU_SET_SCALE1_BASE: // NPU_SET_WEIGHT2_BASE (U85)
634 if (ethosu_is_u65(edev))
635 st.scale[1].base = addr;
636 else
637 st.weight[2].base = addr;
638 break;
639 case NPU_SET_SCALE1_LENGTH: // NPU_SET_WEIGHT2_LENGTH (U85)
640 if (ethosu_is_u65(edev))
641 st.scale[1].length = cmds[1];
642 else
643 st.weight[2].length = cmds[1];
644 break;
645 case NPU_SET_WEIGHT3_BASE:
646 st.weight[3].base = addr;
647 break;
648 case NPU_SET_WEIGHT3_LENGTH:
649 st.weight[3].length = cmds[1];
650 break;
651
652 case NPU_SET_DMA0_SRC_REGION:
653 if (param & 0x100)
654 st.dma.src.region = -1;
655 else
656 st.dma.src.region = param & 0x7;
657 st.dma.mode = (param >> 9) & 0x3;
658 break;
659 case NPU_SET_DMA0_DST_REGION:
660 if (param & 0x100)
661 st.dma.dst.region = -1;
662 else
663 st.dma.dst.region = param & 0x7;
664 break;
665 case NPU_SET_DMA0_SIZE0:
666 st.dma.size0 = param;
667 break;
668 case NPU_SET_DMA0_SIZE1:
669 st.dma.size1 = param;
670 break;
671 case NPU_SET_DMA0_SRC_STRIDE0:
672 st.dma.src.stride[0] = ((s64)addr << 24) >> 24;
673 break;
674 case NPU_SET_DMA0_SRC_STRIDE1:
675 st.dma.src.stride[1] = ((s64)addr << 24) >> 24;
676 break;
677 case NPU_SET_DMA0_DST_STRIDE0:
678 st.dma.dst.stride[0] = ((s64)addr << 24) >> 24;
679 break;
680 case NPU_SET_DMA0_DST_STRIDE1:
681 st.dma.dst.stride[1] = ((s64)addr << 24) >> 24;
682 break;
683 case NPU_SET_DMA0_SRC:
684 st.dma.src.offset = addr;
685 break;
686 case NPU_SET_DMA0_DST:
687 st.dma.dst.offset = addr;
688 break;
689 case NPU_SET_DMA0_LEN:
690 st.dma.src.len = st.dma.dst.len = addr;
691 break;
692 default:
693 break;
694 }
695 }
696
697 if (!ends_with_stop)
698 return -EINVAL;
699
700 for (i = 0; i < NPU_BASEP_REGION_MAX; i++) {
701 if (!info->region_size[i])
702 continue;
703 dev_dbg(ddev->dev, "region %d max size: 0x%llx\n",
704 i, info->region_size[i]);
705 }
706
707 bo->info = no_free_ptr(info);
708 return 0;
709 }
710
711 /**
712 * ethosu_gem_cmdstream_create() - Create a GEM object and attach it to a handle.
713 * @file: DRM file.
714 * @ddev: DRM device.
715 * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared.
716 * @size: Size of the GEM object to allocate.
717 * @flags: Combination of drm_ethosu_bo_flags flags.
718 * @handle: Pointer holding the handle pointing to the new GEM object.
719 *
720 * Return: Zero on success
721 */
ethosu_gem_cmdstream_create(struct drm_file * file,struct drm_device * ddev,u32 size,u64 data,u32 flags,u32 * handle)722 int ethosu_gem_cmdstream_create(struct drm_file *file,
723 struct drm_device *ddev,
724 u32 size, u64 data, u32 flags, u32 *handle)
725 {
726 int ret;
727 struct drm_gem_dma_object *mem;
728 struct ethosu_gem_object *bo;
729
730 mem = drm_gem_dma_create(ddev, size);
731 if (IS_ERR(mem))
732 return PTR_ERR(mem);
733
734 bo = to_ethosu_bo(&mem->base);
735 bo->flags = flags;
736
737 ret = ethosu_gem_cmdstream_copy_and_validate(ddev,
738 (void __user *)(uintptr_t)data,
739 bo, size);
740 if (ret)
741 goto fail;
742
743 /*
744 * Allocate an id of idr table where the obj is registered
745 * and handle has the id what user can see.
746 */
747 ret = drm_gem_handle_create(file, &mem->base, handle);
748
749 fail:
750 /* drop reference from allocate - handle holds it now. */
751 drm_gem_object_put(&mem->base);
752
753 return ret;
754 }
755