1 // SPDX-License-Identifier: GPL-2.0-only or MIT 2 /* Copyright 2025 Arm, Ltd. */ 3 4 #include <linux/err.h> 5 #include <linux/overflow.h> 6 #include <linux/slab.h> 7 8 #include <drm/ethosu_accel.h> 9 10 #include "ethosu_device.h" 11 #include "ethosu_gem.h" 12 13 static void ethosu_gem_free_object(struct drm_gem_object *obj) 14 { 15 struct ethosu_gem_object *bo = to_ethosu_bo(obj); 16 17 kfree(bo->info); 18 drm_gem_free_mmap_offset(&bo->base.base); 19 drm_gem_dma_free(&bo->base); 20 } 21 22 static int ethosu_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 23 { 24 struct ethosu_gem_object *bo = to_ethosu_bo(obj); 25 26 /* Don't allow mmap on objects that have the NO_MMAP flag set. */ 27 if (bo->flags & DRM_ETHOSU_BO_NO_MMAP) 28 return -EINVAL; 29 30 return drm_gem_dma_object_mmap(obj, vma); 31 } 32 33 static const struct drm_gem_object_funcs ethosu_gem_funcs = { 34 .free = ethosu_gem_free_object, 35 .print_info = drm_gem_dma_object_print_info, 36 .get_sg_table = drm_gem_dma_object_get_sg_table, 37 .vmap = drm_gem_dma_object_vmap, 38 .mmap = ethosu_gem_mmap, 39 .vm_ops = &drm_gem_dma_vm_ops, 40 }; 41 42 /** 43 * ethosu_gem_create_object - Implementation of driver->gem_create_object. 44 * @ddev: DRM device 45 * @size: Size in bytes of the memory the object will reference 46 * 47 * This lets the GEM helpers allocate object structs for us, and keep 48 * our BO stats correct. 49 */ 50 struct drm_gem_object *ethosu_gem_create_object(struct drm_device *ddev, size_t size) 51 { 52 struct ethosu_gem_object *obj; 53 54 obj = kzalloc_obj(*obj); 55 if (!obj) 56 return ERR_PTR(-ENOMEM); 57 58 obj->base.base.funcs = ðosu_gem_funcs; 59 return &obj->base.base; 60 } 61 62 /** 63 * ethosu_gem_create_with_handle() - Create a GEM object and attach it to a handle. 64 * @file: DRM file. 65 * @ddev: DRM device. 66 * @size: Size of the GEM object to allocate. 67 * @flags: Combination of drm_ethosu_bo_flags flags. 68 * @handle: Pointer holding the handle pointing to the new GEM object. 69 * 70 * Return: Zero on success 71 */ 72 int ethosu_gem_create_with_handle(struct drm_file *file, 73 struct drm_device *ddev, 74 u64 *size, u32 flags, u32 *handle) 75 { 76 struct drm_gem_dma_object *mem; 77 struct ethosu_gem_object *bo; 78 int ret; 79 80 mem = drm_gem_dma_create(ddev, *size); 81 if (IS_ERR(mem)) 82 return PTR_ERR(mem); 83 84 bo = to_ethosu_bo(&mem->base); 85 bo->flags = flags; 86 87 /* 88 * Allocate an id of idr table where the obj is registered 89 * and handle has the id what user can see. 90 */ 91 ret = drm_gem_handle_create(file, &mem->base, handle); 92 if (!ret) 93 *size = bo->base.base.size; 94 95 /* drop reference from allocate - handle holds it now. */ 96 drm_gem_object_put(&mem->base); 97 98 return ret; 99 } 100 101 struct dma { 102 s8 region; 103 u64 len; 104 u64 offset; 105 s64 stride[2]; 106 }; 107 108 struct dma_state { 109 u16 size0; 110 u16 size1; 111 s8 mode; 112 struct dma src; 113 struct dma dst; 114 }; 115 116 struct buffer { 117 u64 base; 118 u32 length; 119 s8 region; 120 }; 121 122 struct feat_matrix { 123 u64 base[4]; 124 s64 stride_x; 125 s64 stride_y; 126 s64 stride_c; 127 s8 region; 128 u8 broadcast; 129 u16 stride_kernel; 130 u16 precision; 131 u16 depth; 132 u16 width; 133 u16 width0; 134 u16 height[3]; 135 u8 pad_top; 136 u8 pad_left; 137 u8 pad_bottom; 138 u8 pad_right; 139 }; 140 141 struct cmd_state { 142 struct dma_state dma; 143 struct buffer scale[2]; 144 struct buffer weight[4]; 145 struct feat_matrix ofm; 146 struct feat_matrix ifm; 147 struct feat_matrix ifm2; 148 }; 149 150 static void cmd_state_init(struct cmd_state *st) 151 { 152 /* Initialize to all 1s to detect missing setup */ 153 memset(st, 0xff, sizeof(*st)); 154 } 155 156 static u64 cmd_to_addr(u32 *cmd) 157 { 158 return (((u64)cmd[0] & 0xff0000) << 16) | cmd[1]; 159 } 160 161 static u64 dma_length(struct ethosu_validated_cmdstream_info *info, 162 struct dma_state *dma_st, struct dma *dma) 163 { 164 s8 mode = dma_st->mode; 165 u64 len = dma->len; 166 167 if (len == U64_MAX) 168 return U64_MAX; 169 170 if (mode >= 1) { 171 if (dma->stride[0] < 0 && (u64)(-dma->stride[0]) > len) 172 return U64_MAX; 173 len += dma->stride[0]; 174 if (check_mul_overflow(len, (u64)dma_st->size0, &len)) 175 return U64_MAX; 176 } 177 if (mode == 2) { 178 if (dma->stride[1] < 0 && (u64)(-dma->stride[1]) > len) 179 return U64_MAX; 180 len += dma->stride[1]; 181 if (check_mul_overflow(len, (u64)dma_st->size1, &len)) 182 return U64_MAX; 183 } 184 if (dma->region >= 0) { 185 u64 end; 186 187 if (check_add_overflow(len, dma->offset, &end)) 188 return U64_MAX; 189 info->region_size[dma->region] = max(info->region_size[dma->region], end); 190 } 191 192 return len; 193 } 194 195 static bool feat_matrix_chained(struct ethosu_device *edev, struct feat_matrix *fm) 196 { 197 u32 storage = fm->precision >> 14; 198 199 return !ethosu_is_u65(edev) && storage == 2; 200 } 201 202 static u64 feat_matrix_length(struct ethosu_device *edev, 203 struct ethosu_validated_cmdstream_info *info, 204 struct feat_matrix *fm, 205 u32 x, u32 y, u32 c, bool ofm) 206 { 207 u32 element_size, storage = ethosu_is_u65(edev) ? 0 : fm->precision >> 14; 208 int tile = 0; 209 u64 addr; 210 211 if (fm->region < 0) 212 return U64_MAX; 213 214 if (feat_matrix_chained(edev, fm)) 215 return 0; 216 217 switch (storage) { 218 case 0: 219 if (x >= fm->width0 + 1) { 220 x -= fm->width0 + 1; 221 tile += 1; 222 } 223 if (y >= fm->height[tile] + 1) { 224 y -= fm->height[tile] + 1; 225 tile += 2; 226 } 227 break; 228 case 1: 229 if (y >= fm->height[1] + 1) { 230 y -= fm->height[1] + 1; 231 tile = 2; 232 } else if (y >= fm->height[0] + 1) { 233 y -= fm->height[0] + 1; 234 tile = 1; 235 } 236 break; 237 default: 238 return U64_MAX; 239 } 240 if (fm->base[tile] == U64_MAX) 241 return U64_MAX; 242 243 addr = fm->base[tile] + y * fm->stride_y; 244 245 switch ((fm->precision >> 6) & 0x3) { // format 246 case 0: //nhwc: 247 element_size = BIT((fm->precision >> (ofm ? 1 : 2)) & 0x3); 248 addr += x * fm->stride_x + c * element_size; 249 break; 250 case 1: //nhcwb16: 251 element_size = BIT((fm->precision >> (ofm ? 1 : 2)) & 0x3); 252 253 addr += (c / 16) * fm->stride_c + (16 * x + (c & 0xf)) * element_size; 254 break; 255 } 256 257 info->region_size[fm->region] = max(info->region_size[fm->region], addr + 1); 258 259 return addr; 260 } 261 262 static int calc_sizes(struct drm_device *ddev, 263 struct ethosu_validated_cmdstream_info *info, 264 u16 op, struct cmd_state *st, 265 bool ifm, bool ifm2, bool weight, bool scale) 266 { 267 struct ethosu_device *edev = to_ethosu_device(ddev); 268 u64 len; 269 270 if (ifm) { 271 if (st->ifm.stride_kernel == U16_MAX) 272 return -EINVAL; 273 u32 stride_y = ((st->ifm.stride_kernel >> 8) & 0x2) + 274 ((st->ifm.stride_kernel >> 1) & 0x1) + 1; 275 u32 stride_x = ((st->ifm.stride_kernel >> 5) & 0x2) + 276 (st->ifm.stride_kernel & 0x1) + 1; 277 s32 ifm_height = st->ofm.height[2] * stride_y + 278 st->ifm.height[2] - (st->ifm.pad_top + st->ifm.pad_bottom); 279 s32 ifm_width = st->ofm.width * stride_x + 280 st->ifm.width - (st->ifm.pad_left + st->ifm.pad_right); 281 282 if (ifm_height < 0 || ifm_width < 0) 283 return -EINVAL; 284 285 len = feat_matrix_length(edev, info, &st->ifm, ifm_width, 286 ifm_height, st->ifm.depth, false); 287 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n", 288 op, st->ifm.region, st->ifm.base[0], len); 289 if (len == U64_MAX) 290 return -EINVAL; 291 } 292 293 if (ifm2) { 294 len = feat_matrix_length(edev, info, &st->ifm2, st->ifm.depth, 295 0, st->ofm.depth, false); 296 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n", 297 op, st->ifm2.region, st->ifm2.base[0], len); 298 if (len == U64_MAX) 299 return -EINVAL; 300 } 301 302 if (weight) { 303 dev_dbg(ddev->dev, "op %d: W:%d:0x%llx-0x%llx\n", 304 op, st->weight[0].region, st->weight[0].base, 305 st->weight[0].base + st->weight[0].length - 1); 306 if (st->weight[0].region < 0 || st->weight[0].base == U64_MAX || 307 st->weight[0].length == U32_MAX) 308 return -EINVAL; 309 info->region_size[st->weight[0].region] = 310 max(info->region_size[st->weight[0].region], 311 st->weight[0].base + st->weight[0].length); 312 } 313 314 if (scale) { 315 dev_dbg(ddev->dev, "op %d: S:%d:0x%llx-0x%llx\n", 316 op, st->scale[0].region, st->scale[0].base, 317 st->scale[0].base + st->scale[0].length - 1); 318 if (st->scale[0].region < 0 || st->scale[0].base == U64_MAX || 319 st->scale[0].length == U32_MAX) 320 return -EINVAL; 321 info->region_size[st->scale[0].region] = 322 max(info->region_size[st->scale[0].region], 323 st->scale[0].base + st->scale[0].length); 324 } 325 326 len = feat_matrix_length(edev, info, &st->ofm, st->ofm.width, 327 st->ofm.height[2], st->ofm.depth, true); 328 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n", 329 op, st->ofm.region, st->ofm.base[0], len); 330 if (len == U64_MAX) 331 return -EINVAL; 332 if (!feat_matrix_chained(edev, &st->ofm)) 333 info->output_region[st->ofm.region] = true; 334 335 return 0; 336 } 337 338 static int calc_sizes_elemwise(struct drm_device *ddev, 339 struct ethosu_validated_cmdstream_info *info, 340 u16 op, struct cmd_state *st, 341 bool ifm, bool ifm2) 342 { 343 struct ethosu_device *edev = to_ethosu_device(ddev); 344 u32 height, width, depth; 345 u64 len; 346 347 if (ifm) { 348 height = st->ifm.broadcast & 0x1 ? 0 : st->ofm.height[2]; 349 width = st->ifm.broadcast & 0x2 ? 0 : st->ofm.width; 350 depth = st->ifm.broadcast & 0x4 ? 0 : st->ofm.depth; 351 352 len = feat_matrix_length(edev, info, &st->ifm, width, 353 height, depth, false); 354 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n", 355 op, st->ifm.region, st->ifm.base[0], len); 356 if (len == U64_MAX) 357 return -EINVAL; 358 } 359 360 if (ifm2) { 361 height = st->ifm2.broadcast & 0x1 ? 0 : st->ofm.height[2]; 362 width = st->ifm2.broadcast & 0x2 ? 0 : st->ofm.width; 363 depth = st->ifm2.broadcast & 0x4 ? 0 : st->ofm.depth; 364 365 len = feat_matrix_length(edev, info, &st->ifm2, width, 366 height, depth, false); 367 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n", 368 op, st->ifm2.region, st->ifm2.base[0], len); 369 if (len == U64_MAX) 370 return -EINVAL; 371 } 372 373 len = feat_matrix_length(edev, info, &st->ofm, st->ofm.width, 374 st->ofm.height[2], st->ofm.depth, true); 375 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n", 376 op, st->ofm.region, st->ofm.base[0], len); 377 if (len == U64_MAX) 378 return -EINVAL; 379 if (!feat_matrix_chained(edev, &st->ofm)) 380 info->output_region[st->ofm.region] = true; 381 382 return 0; 383 } 384 385 static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev, 386 u32 __user *ucmds, 387 struct ethosu_gem_object *bo, 388 u32 size) 389 { 390 struct ethosu_validated_cmdstream_info __free(kfree) *info = kzalloc_obj(*info); 391 struct ethosu_device *edev = to_ethosu_device(ddev); 392 u32 *bocmds = bo->base.vaddr; 393 bool ends_with_stop = false; 394 struct cmd_state st; 395 int i, ret; 396 397 if (!info) 398 return -ENOMEM; 399 info->cmd_size = size; 400 401 cmd_state_init(&st); 402 403 for (i = 0; i < size / 4; i++) { 404 bool use_ifm, use_ifm2, use_scale; 405 u64 dstlen, srclen; 406 u16 cmd, param; 407 u32 cmds[2]; 408 u64 addr; 409 410 if (get_user(cmds[0], ucmds++)) 411 return -EFAULT; 412 413 bocmds[i] = cmds[0]; 414 415 cmd = cmds[0]; 416 param = cmds[0] >> 16; 417 418 if (cmd & 0x4000) { 419 if (get_user(cmds[1], ucmds++)) 420 return -EFAULT; 421 422 i++; 423 if (i >= size / 4) 424 return -EINVAL; 425 bocmds[i] = cmds[1]; 426 addr = cmd_to_addr(cmds); 427 } 428 429 switch (cmd) { 430 case NPU_OP_STOP: 431 if (i != size / 4 - 1) 432 return -EINVAL; 433 ends_with_stop = true; 434 break; 435 case NPU_OP_DMA_START: 436 srclen = dma_length(info, &st.dma, &st.dma.src); 437 dstlen = dma_length(info, &st.dma, &st.dma.dst); 438 if (srclen == U64_MAX || dstlen == U64_MAX) 439 return -EINVAL; 440 441 if (st.dma.dst.region >= 0) 442 info->output_region[st.dma.dst.region] = true; 443 dev_dbg(ddev->dev, "cmd: DMA SRC:%d:0x%llx+0x%llx DST:%d:0x%llx+0x%llx\n", 444 st.dma.src.region, st.dma.src.offset, srclen, 445 st.dma.dst.region, st.dma.dst.offset, dstlen); 446 break; 447 case NPU_OP_CONV: 448 case NPU_OP_DEPTHWISE: 449 use_ifm2 = param & 0x1; // weights_ifm2 450 use_scale = !(st.ofm.precision & 0x100); 451 ret = calc_sizes(ddev, info, cmd, &st, true, use_ifm2, 452 !use_ifm2, use_scale); 453 if (ret) 454 return ret; 455 break; 456 case NPU_OP_POOL: 457 use_ifm = param != 0x4; // pooling mode 458 use_scale = !(st.ofm.precision & 0x100); 459 ret = calc_sizes(ddev, info, cmd, &st, use_ifm, false, 460 false, use_scale); 461 if (ret) 462 return ret; 463 break; 464 case NPU_OP_ELEMENTWISE: 465 use_scale = ethosu_is_u65(edev) ? 466 (st.ifm2.broadcast & 0x80) : 467 (st.ifm2.broadcast == 8); 468 use_ifm2 = !(use_scale || (param == 5) || 469 (param == 6) || (param == 7) || (param == 0x24)); 470 use_ifm = st.ifm.broadcast != 8; 471 ret = calc_sizes_elemwise(ddev, info, cmd, &st, use_ifm, use_ifm2); 472 if (ret) 473 return ret; 474 break; 475 case NPU_OP_RESIZE: // U85 only 476 return -EINVAL; 477 case NPU_SET_KERNEL_WIDTH_M1: 478 st.ifm.width = param; 479 break; 480 case NPU_SET_KERNEL_HEIGHT_M1: 481 st.ifm.height[2] = param; 482 break; 483 case NPU_SET_KERNEL_STRIDE: 484 st.ifm.stride_kernel = param; 485 break; 486 case NPU_SET_IFM_PAD_TOP: 487 st.ifm.pad_top = param & 0x7f; 488 break; 489 case NPU_SET_IFM_PAD_LEFT: 490 st.ifm.pad_left = param & 0x7f; 491 break; 492 case NPU_SET_IFM_PAD_RIGHT: 493 st.ifm.pad_right = param & 0xff; 494 break; 495 case NPU_SET_IFM_PAD_BOTTOM: 496 st.ifm.pad_bottom = param & 0xff; 497 break; 498 case NPU_SET_IFM_DEPTH_M1: 499 st.ifm.depth = param; 500 break; 501 case NPU_SET_IFM_PRECISION: 502 st.ifm.precision = param; 503 break; 504 case NPU_SET_IFM_BROADCAST: 505 st.ifm.broadcast = param; 506 break; 507 case NPU_SET_IFM_REGION: 508 st.ifm.region = param & 0x7; 509 break; 510 case NPU_SET_IFM_WIDTH0_M1: 511 st.ifm.width0 = param; 512 break; 513 case NPU_SET_IFM_HEIGHT0_M1: 514 st.ifm.height[0] = param; 515 break; 516 case NPU_SET_IFM_HEIGHT1_M1: 517 st.ifm.height[1] = param; 518 break; 519 case NPU_SET_IFM_BASE0: 520 case NPU_SET_IFM_BASE1: 521 case NPU_SET_IFM_BASE2: 522 case NPU_SET_IFM_BASE3: 523 st.ifm.base[cmd & 0x3] = addr; 524 break; 525 case NPU_SET_IFM_STRIDE_X: 526 st.ifm.stride_x = addr; 527 break; 528 case NPU_SET_IFM_STRIDE_Y: 529 st.ifm.stride_y = addr; 530 break; 531 case NPU_SET_IFM_STRIDE_C: 532 st.ifm.stride_c = addr; 533 break; 534 535 case NPU_SET_OFM_WIDTH_M1: 536 st.ofm.width = param; 537 break; 538 case NPU_SET_OFM_HEIGHT_M1: 539 st.ofm.height[2] = param; 540 break; 541 case NPU_SET_OFM_DEPTH_M1: 542 st.ofm.depth = param; 543 break; 544 case NPU_SET_OFM_PRECISION: 545 st.ofm.precision = param; 546 break; 547 case NPU_SET_OFM_REGION: 548 st.ofm.region = param & 0x7; 549 break; 550 case NPU_SET_OFM_WIDTH0_M1: 551 st.ofm.width0 = param; 552 break; 553 case NPU_SET_OFM_HEIGHT0_M1: 554 st.ofm.height[0] = param; 555 break; 556 case NPU_SET_OFM_HEIGHT1_M1: 557 st.ofm.height[1] = param; 558 break; 559 case NPU_SET_OFM_BASE0: 560 case NPU_SET_OFM_BASE1: 561 case NPU_SET_OFM_BASE2: 562 case NPU_SET_OFM_BASE3: 563 st.ofm.base[cmd & 0x3] = addr; 564 break; 565 case NPU_SET_OFM_STRIDE_X: 566 st.ofm.stride_x = addr; 567 break; 568 case NPU_SET_OFM_STRIDE_Y: 569 st.ofm.stride_y = addr; 570 break; 571 case NPU_SET_OFM_STRIDE_C: 572 st.ofm.stride_c = addr; 573 break; 574 575 case NPU_SET_IFM2_BROADCAST: 576 st.ifm2.broadcast = param; 577 break; 578 case NPU_SET_IFM2_PRECISION: 579 st.ifm2.precision = param; 580 break; 581 case NPU_SET_IFM2_REGION: 582 st.ifm2.region = param & 0x7; 583 break; 584 case NPU_SET_IFM2_WIDTH0_M1: 585 st.ifm2.width0 = param; 586 break; 587 case NPU_SET_IFM2_HEIGHT0_M1: 588 st.ifm2.height[0] = param; 589 break; 590 case NPU_SET_IFM2_HEIGHT1_M1: 591 st.ifm2.height[1] = param; 592 break; 593 case NPU_SET_IFM2_BASE0: 594 case NPU_SET_IFM2_BASE1: 595 case NPU_SET_IFM2_BASE2: 596 case NPU_SET_IFM2_BASE3: 597 st.ifm2.base[cmd & 0x3] = addr; 598 break; 599 case NPU_SET_IFM2_STRIDE_X: 600 st.ifm2.stride_x = addr; 601 break; 602 case NPU_SET_IFM2_STRIDE_Y: 603 st.ifm2.stride_y = addr; 604 break; 605 case NPU_SET_IFM2_STRIDE_C: 606 st.ifm2.stride_c = addr; 607 break; 608 609 case NPU_SET_WEIGHT_REGION: 610 st.weight[0].region = param & 0x7; 611 break; 612 case NPU_SET_SCALE_REGION: 613 st.scale[0].region = param & 0x7; 614 break; 615 case NPU_SET_WEIGHT_BASE: 616 st.weight[0].base = addr; 617 break; 618 case NPU_SET_WEIGHT_LENGTH: 619 st.weight[0].length = cmds[1]; 620 break; 621 case NPU_SET_SCALE_BASE: 622 st.scale[0].base = addr; 623 break; 624 case NPU_SET_SCALE_LENGTH: 625 st.scale[0].length = cmds[1]; 626 break; 627 case NPU_SET_WEIGHT1_BASE: 628 st.weight[1].base = addr; 629 break; 630 case NPU_SET_WEIGHT1_LENGTH: 631 st.weight[1].length = cmds[1]; 632 break; 633 case NPU_SET_SCALE1_BASE: // NPU_SET_WEIGHT2_BASE (U85) 634 if (ethosu_is_u65(edev)) 635 st.scale[1].base = addr; 636 else 637 st.weight[2].base = addr; 638 break; 639 case NPU_SET_SCALE1_LENGTH: // NPU_SET_WEIGHT2_LENGTH (U85) 640 if (ethosu_is_u65(edev)) 641 st.scale[1].length = cmds[1]; 642 else 643 st.weight[2].length = cmds[1]; 644 break; 645 case NPU_SET_WEIGHT3_BASE: 646 st.weight[3].base = addr; 647 break; 648 case NPU_SET_WEIGHT3_LENGTH: 649 st.weight[3].length = cmds[1]; 650 break; 651 652 case NPU_SET_DMA0_SRC_REGION: 653 if (param & 0x100) 654 st.dma.src.region = -1; 655 else 656 st.dma.src.region = param & 0x7; 657 st.dma.mode = (param >> 9) & 0x3; 658 break; 659 case NPU_SET_DMA0_DST_REGION: 660 if (param & 0x100) 661 st.dma.dst.region = -1; 662 else 663 st.dma.dst.region = param & 0x7; 664 break; 665 case NPU_SET_DMA0_SIZE0: 666 st.dma.size0 = param; 667 break; 668 case NPU_SET_DMA0_SIZE1: 669 st.dma.size1 = param; 670 break; 671 case NPU_SET_DMA0_SRC_STRIDE0: 672 st.dma.src.stride[0] = ((s64)addr << 24) >> 24; 673 break; 674 case NPU_SET_DMA0_SRC_STRIDE1: 675 st.dma.src.stride[1] = ((s64)addr << 24) >> 24; 676 break; 677 case NPU_SET_DMA0_DST_STRIDE0: 678 st.dma.dst.stride[0] = ((s64)addr << 24) >> 24; 679 break; 680 case NPU_SET_DMA0_DST_STRIDE1: 681 st.dma.dst.stride[1] = ((s64)addr << 24) >> 24; 682 break; 683 case NPU_SET_DMA0_SRC: 684 st.dma.src.offset = addr; 685 break; 686 case NPU_SET_DMA0_DST: 687 st.dma.dst.offset = addr; 688 break; 689 case NPU_SET_DMA0_LEN: 690 st.dma.src.len = st.dma.dst.len = addr; 691 break; 692 default: 693 break; 694 } 695 } 696 697 if (!ends_with_stop) 698 return -EINVAL; 699 700 for (i = 0; i < NPU_BASEP_REGION_MAX; i++) { 701 if (!info->region_size[i]) 702 continue; 703 dev_dbg(ddev->dev, "region %d max size: 0x%llx\n", 704 i, info->region_size[i]); 705 } 706 707 bo->info = no_free_ptr(info); 708 return 0; 709 } 710 711 /** 712 * ethosu_gem_cmdstream_create() - Create a GEM object and attach it to a handle. 713 * @file: DRM file. 714 * @ddev: DRM device. 715 * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared. 716 * @size: Size of the GEM object to allocate. 717 * @flags: Combination of drm_ethosu_bo_flags flags. 718 * @handle: Pointer holding the handle pointing to the new GEM object. 719 * 720 * Return: Zero on success 721 */ 722 int ethosu_gem_cmdstream_create(struct drm_file *file, 723 struct drm_device *ddev, 724 u32 size, u64 data, u32 flags, u32 *handle) 725 { 726 int ret; 727 struct drm_gem_dma_object *mem; 728 struct ethosu_gem_object *bo; 729 730 mem = drm_gem_dma_create(ddev, size); 731 if (IS_ERR(mem)) 732 return PTR_ERR(mem); 733 734 bo = to_ethosu_bo(&mem->base); 735 bo->flags = flags; 736 737 ret = ethosu_gem_cmdstream_copy_and_validate(ddev, 738 (void __user *)(uintptr_t)data, 739 bo, size); 740 if (ret) 741 goto fail; 742 743 /* 744 * Allocate an id of idr table where the obj is registered 745 * and handle has the id what user can see. 746 */ 747 ret = drm_gem_handle_create(file, &mem->base, handle); 748 749 fail: 750 /* drop reference from allocate - handle holds it now. */ 751 drm_gem_object_put(&mem->base); 752 753 return ret; 754 } 755