1 // SPDX-License-Identifier: GPL-2.0-only or MIT 2 /* Copyright 2025 Arm, Ltd. */ 3 4 #include <linux/err.h> 5 #include <linux/overflow.h> 6 #include <linux/slab.h> 7 8 #include <drm/ethosu_accel.h> 9 10 #include "ethosu_device.h" 11 #include "ethosu_gem.h" 12 13 static void ethosu_gem_free_object(struct drm_gem_object *obj) 14 { 15 struct ethosu_gem_object *bo = to_ethosu_bo(obj); 16 17 kfree(bo->info); 18 drm_gem_free_mmap_offset(&bo->base.base); 19 drm_gem_dma_free(&bo->base); 20 } 21 22 static int ethosu_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 23 { 24 struct ethosu_gem_object *bo = to_ethosu_bo(obj); 25 26 /* Don't allow mmap on objects that have the NO_MMAP flag set. */ 27 if (bo->flags & DRM_ETHOSU_BO_NO_MMAP) 28 return -EINVAL; 29 30 return drm_gem_dma_object_mmap(obj, vma); 31 } 32 33 static const struct drm_gem_object_funcs ethosu_gem_funcs = { 34 .free = ethosu_gem_free_object, 35 .print_info = drm_gem_dma_object_print_info, 36 .get_sg_table = drm_gem_dma_object_get_sg_table, 37 .vmap = drm_gem_dma_object_vmap, 38 .mmap = ethosu_gem_mmap, 39 .vm_ops = &drm_gem_dma_vm_ops, 40 }; 41 42 /** 43 * ethosu_gem_create_object - Implementation of driver->gem_create_object. 44 * @ddev: DRM device 45 * @size: Size in bytes of the memory the object will reference 46 * 47 * This lets the GEM helpers allocate object structs for us, and keep 48 * our BO stats correct. 49 */ 50 struct drm_gem_object *ethosu_gem_create_object(struct drm_device *ddev, size_t size) 51 { 52 struct ethosu_gem_object *obj; 53 54 obj = kzalloc_obj(*obj); 55 if (!obj) 56 return ERR_PTR(-ENOMEM); 57 58 obj->base.base.funcs = ðosu_gem_funcs; 59 return &obj->base.base; 60 } 61 62 /** 63 * ethosu_gem_create_with_handle() - Create a GEM object and attach it to a handle. 64 * @file: DRM file. 65 * @ddev: DRM device. 66 * @size: Size of the GEM object to allocate. 67 * @flags: Combination of drm_ethosu_bo_flags flags. 68 * @handle: Pointer holding the handle pointing to the new GEM object. 69 * 70 * Return: Zero on success 71 */ 72 int ethosu_gem_create_with_handle(struct drm_file *file, 73 struct drm_device *ddev, 74 u64 *size, u32 flags, u32 *handle) 75 { 76 struct drm_gem_dma_object *mem; 77 struct ethosu_gem_object *bo; 78 int ret; 79 80 mem = drm_gem_dma_create(ddev, *size); 81 if (IS_ERR(mem)) 82 return PTR_ERR(mem); 83 84 bo = to_ethosu_bo(&mem->base); 85 bo->flags = flags; 86 87 /* 88 * Allocate an id of idr table where the obj is registered 89 * and handle has the id what user can see. 90 */ 91 ret = drm_gem_handle_create(file, &mem->base, handle); 92 if (!ret) 93 *size = bo->base.base.size; 94 95 /* drop reference from allocate - handle holds it now. */ 96 drm_gem_object_put(&mem->base); 97 98 return ret; 99 } 100 101 struct dma { 102 s8 region; 103 u64 len; 104 u64 offset; 105 s64 stride[2]; 106 }; 107 108 struct dma_state { 109 u16 size0; 110 u16 size1; 111 s8 mode; 112 struct dma src; 113 struct dma dst; 114 }; 115 116 struct buffer { 117 u64 base; 118 u32 length; 119 s8 region; 120 }; 121 122 struct feat_matrix { 123 u64 base[4]; 124 s64 stride_x; 125 s64 stride_y; 126 s64 stride_c; 127 s8 region; 128 u8 broadcast; 129 u16 stride_kernel; 130 u16 precision; 131 u16 depth; 132 u16 width; 133 u16 width0; 134 u16 height[3]; 135 u8 pad_top; 136 u8 pad_left; 137 u8 pad_bottom; 138 u8 pad_right; 139 }; 140 141 struct cmd_state { 142 struct dma_state dma; 143 struct buffer scale[2]; 144 struct buffer weight[4]; 145 struct feat_matrix ofm; 146 struct feat_matrix ifm; 147 struct feat_matrix ifm2; 148 }; 149 150 static void cmd_state_init(struct cmd_state *st) 151 { 152 /* Initialize to all 1s to detect missing setup */ 153 memset(st, 0xff, sizeof(*st)); 154 } 155 156 static u64 cmd_to_addr(u32 *cmd) 157 { 158 return (((u64)cmd[0] & 0xff0000) << 16) | cmd[1]; 159 } 160 161 static u64 dma_length(struct ethosu_validated_cmdstream_info *info, 162 struct dma_state *dma_st, struct dma *dma) 163 { 164 s8 mode = dma_st->mode; 165 u64 len = dma->len; 166 167 if (len == U64_MAX) 168 return U64_MAX; 169 170 if (mode >= 1) { 171 if (dma->stride[0] < 0 && (u64)(-dma->stride[0]) > len) 172 return U64_MAX; 173 len += dma->stride[0]; 174 if (check_mul_overflow(len, (u64)dma_st->size0, &len)) 175 return U64_MAX; 176 } 177 if (mode == 2) { 178 if (dma->stride[1] < 0 && (u64)(-dma->stride[1]) > len) 179 return U64_MAX; 180 len += dma->stride[1]; 181 if (check_mul_overflow(len, (u64)dma_st->size1, &len)) 182 return U64_MAX; 183 } 184 if (dma->region >= 0) { 185 u64 end; 186 187 if (check_add_overflow(len, dma->offset, &end)) 188 return U64_MAX; 189 info->region_size[dma->region] = max(info->region_size[dma->region], end); 190 } 191 192 return len; 193 } 194 195 static bool feat_matrix_chained(struct ethosu_device *edev, struct feat_matrix *fm) 196 { 197 u32 storage = fm->precision >> 14; 198 199 return !ethosu_is_u65(edev) && storage == 2; 200 } 201 202 static u64 feat_matrix_length(struct ethosu_device *edev, 203 struct ethosu_validated_cmdstream_info *info, 204 struct feat_matrix *fm, 205 u32 x, u32 y, u32 c, bool ofm) 206 { 207 u32 element_size, storage = fm->precision >> 14; 208 int tile = 0; 209 u64 addr; 210 211 if (fm->region < 0) 212 return U64_MAX; 213 214 if (feat_matrix_chained(edev, fm)) 215 return 0; 216 217 switch (storage) { 218 case 0: 219 if (x >= fm->width0 + 1) { 220 x -= fm->width0 + 1; 221 tile += 1; 222 } 223 if (y >= fm->height[tile] + 1) { 224 y -= fm->height[tile] + 1; 225 tile += 2; 226 } 227 break; 228 case 1: 229 if (y >= fm->height[1] + 1) { 230 y -= fm->height[1] + 1; 231 tile = 2; 232 } else if (y >= fm->height[0] + 1) { 233 y -= fm->height[0] + 1; 234 tile = 1; 235 } 236 break; 237 default: 238 return U64_MAX; 239 } 240 if (fm->base[tile] == U64_MAX) 241 return U64_MAX; 242 243 addr = fm->base[tile] + y * fm->stride_y; 244 245 switch ((fm->precision >> 6) & 0x3) { // format 246 case 0: //nhwc: 247 element_size = BIT((fm->precision >> (ofm ? 1 : 2)) & 0x3); 248 addr += x * fm->stride_x + c * element_size; 249 break; 250 case 1: //nhcwb16: 251 element_size = BIT((fm->precision >> (ofm ? 1 : 2)) & 0x3); 252 253 addr += (c / 16) * fm->stride_c + (16 * x + (c & 0xf)) * element_size; 254 break; 255 } 256 257 info->region_size[fm->region] = max(info->region_size[fm->region], addr + 1); 258 259 return addr; 260 } 261 262 static int calc_sizes(struct drm_device *ddev, 263 struct ethosu_validated_cmdstream_info *info, 264 u16 op, struct cmd_state *st, 265 bool ifm, bool ifm2, bool weight, bool scale) 266 { 267 struct ethosu_device *edev = to_ethosu_device(ddev); 268 u64 len; 269 270 if (ifm) { 271 if (st->ifm.stride_kernel == U16_MAX) 272 return -EINVAL; 273 u32 stride_y = ((st->ifm.stride_kernel >> 8) & 0x2) + 274 ((st->ifm.stride_kernel >> 1) & 0x1) + 1; 275 u32 stride_x = ((st->ifm.stride_kernel >> 5) & 0x2) + 276 (st->ifm.stride_kernel & 0x1) + 1; 277 s32 ifm_height = st->ofm.height[2] * stride_y + 278 st->ifm.height[2] - (st->ifm.pad_top + st->ifm.pad_bottom); 279 s32 ifm_width = st->ofm.width * stride_x + 280 st->ifm.width - (st->ifm.pad_left + st->ifm.pad_right); 281 282 if (ifm_height < 0 || ifm_width < 0) 283 return -EINVAL; 284 285 len = feat_matrix_length(edev, info, &st->ifm, ifm_width, 286 ifm_height, st->ifm.depth, false); 287 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n", 288 op, st->ifm.region, st->ifm.base[0], len); 289 if (len == U64_MAX) 290 return -EINVAL; 291 } 292 293 if (ifm2) { 294 len = feat_matrix_length(edev, info, &st->ifm2, st->ifm.depth, 295 0, st->ofm.depth, false); 296 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n", 297 op, st->ifm2.region, st->ifm2.base[0], len); 298 if (len == U64_MAX) 299 return -EINVAL; 300 } 301 302 if (weight) { 303 dev_dbg(ddev->dev, "op %d: W:%d:0x%llx-0x%llx\n", 304 op, st->weight[0].region, st->weight[0].base, 305 st->weight[0].base + st->weight[0].length - 1); 306 if (st->weight[0].region < 0 || st->weight[0].base == U64_MAX || 307 st->weight[0].length == U32_MAX) 308 return -EINVAL; 309 info->region_size[st->weight[0].region] = 310 max(info->region_size[st->weight[0].region], 311 st->weight[0].base + st->weight[0].length); 312 } 313 314 if (scale) { 315 dev_dbg(ddev->dev, "op %d: S:%d:0x%llx-0x%llx\n", 316 op, st->scale[0].region, st->scale[0].base, 317 st->scale[0].base + st->scale[0].length - 1); 318 if (st->scale[0].region < 0 || st->scale[0].base == U64_MAX || 319 st->scale[0].length == U32_MAX) 320 return -EINVAL; 321 info->region_size[st->scale[0].region] = 322 max(info->region_size[st->scale[0].region], 323 st->scale[0].base + st->scale[0].length); 324 } 325 326 len = feat_matrix_length(edev, info, &st->ofm, st->ofm.width, 327 st->ofm.height[2], st->ofm.depth, true); 328 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n", 329 op, st->ofm.region, st->ofm.base[0], len); 330 if (len == U64_MAX) 331 return -EINVAL; 332 if (!feat_matrix_chained(edev, &st->ofm)) 333 info->output_region[st->ofm.region] = true; 334 335 return 0; 336 } 337 338 static int calc_sizes_elemwise(struct drm_device *ddev, 339 struct ethosu_validated_cmdstream_info *info, 340 u16 op, struct cmd_state *st, 341 bool ifm, bool ifm2) 342 { 343 struct ethosu_device *edev = to_ethosu_device(ddev); 344 u32 height, width, depth; 345 u64 len; 346 347 if (ifm) { 348 height = st->ifm.broadcast & 0x1 ? 0 : st->ofm.height[2]; 349 width = st->ifm.broadcast & 0x2 ? 0 : st->ofm.width; 350 depth = st->ifm.broadcast & 0x4 ? 0 : st->ofm.depth; 351 352 len = feat_matrix_length(edev, info, &st->ifm, width, 353 height, depth, false); 354 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n", 355 op, st->ifm.region, st->ifm.base[0], len); 356 if (len == U64_MAX) 357 return -EINVAL; 358 } 359 360 if (ifm2) { 361 height = st->ifm2.broadcast & 0x1 ? 0 : st->ofm.height[2]; 362 width = st->ifm2.broadcast & 0x2 ? 0 : st->ofm.width; 363 depth = st->ifm2.broadcast & 0x4 ? 0 : st->ofm.depth; 364 365 len = feat_matrix_length(edev, info, &st->ifm2, width, 366 height, depth, false); 367 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n", 368 op, st->ifm2.region, st->ifm2.base[0], len); 369 if (len == U64_MAX) 370 return -EINVAL; 371 } 372 373 len = feat_matrix_length(edev, info, &st->ofm, st->ofm.width, 374 st->ofm.height[2], st->ofm.depth, true); 375 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n", 376 op, st->ofm.region, st->ofm.base[0], len); 377 if (len == U64_MAX) 378 return -EINVAL; 379 if (!feat_matrix_chained(edev, &st->ofm)) 380 info->output_region[st->ofm.region] = true; 381 382 return 0; 383 } 384 385 static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev, 386 u32 __user *ucmds, 387 struct ethosu_gem_object *bo, 388 u32 size) 389 { 390 struct ethosu_validated_cmdstream_info __free(kfree) *info = kzalloc_obj(*info); 391 struct ethosu_device *edev = to_ethosu_device(ddev); 392 u32 *bocmds = bo->base.vaddr; 393 struct cmd_state st; 394 int i, ret; 395 396 if (!info) 397 return -ENOMEM; 398 info->cmd_size = size; 399 400 cmd_state_init(&st); 401 402 for (i = 0; i < size / 4; i++) { 403 bool use_ifm, use_ifm2, use_scale; 404 u64 dstlen, srclen; 405 u16 cmd, param; 406 u32 cmds[2]; 407 u64 addr; 408 409 if (get_user(cmds[0], ucmds++)) 410 return -EFAULT; 411 412 bocmds[i] = cmds[0]; 413 414 cmd = cmds[0]; 415 param = cmds[0] >> 16; 416 417 if (cmd & 0x4000) { 418 if (get_user(cmds[1], ucmds++)) 419 return -EFAULT; 420 421 i++; 422 if (i >= size / 4) 423 return -EINVAL; 424 bocmds[i] = cmds[1]; 425 addr = cmd_to_addr(cmds); 426 } 427 428 switch (cmd) { 429 case NPU_OP_DMA_START: 430 srclen = dma_length(info, &st.dma, &st.dma.src); 431 dstlen = dma_length(info, &st.dma, &st.dma.dst); 432 if (srclen == U64_MAX || dstlen == U64_MAX) 433 return -EINVAL; 434 435 if (st.dma.dst.region >= 0) 436 info->output_region[st.dma.dst.region] = true; 437 dev_dbg(ddev->dev, "cmd: DMA SRC:%d:0x%llx+0x%llx DST:%d:0x%llx+0x%llx\n", 438 st.dma.src.region, st.dma.src.offset, srclen, 439 st.dma.dst.region, st.dma.dst.offset, dstlen); 440 break; 441 case NPU_OP_CONV: 442 case NPU_OP_DEPTHWISE: 443 use_ifm2 = param & 0x1; // weights_ifm2 444 use_scale = !(st.ofm.precision & 0x100); 445 ret = calc_sizes(ddev, info, cmd, &st, true, use_ifm2, 446 !use_ifm2, use_scale); 447 if (ret) 448 return ret; 449 break; 450 case NPU_OP_POOL: 451 use_ifm = param != 0x4; // pooling mode 452 use_scale = !(st.ofm.precision & 0x100); 453 ret = calc_sizes(ddev, info, cmd, &st, use_ifm, false, 454 false, use_scale); 455 if (ret) 456 return ret; 457 break; 458 case NPU_OP_ELEMENTWISE: 459 use_scale = ethosu_is_u65(edev) ? 460 (st.ifm2.broadcast & 0x80) : 461 (st.ifm2.broadcast == 8); 462 use_ifm2 = !(use_scale || (param == 5) || 463 (param == 6) || (param == 7) || (param == 0x24)); 464 use_ifm = st.ifm.broadcast != 8; 465 ret = calc_sizes_elemwise(ddev, info, cmd, &st, use_ifm, use_ifm2); 466 if (ret) 467 return ret; 468 break; 469 case NPU_OP_RESIZE: // U85 only 470 return -EINVAL; 471 case NPU_SET_KERNEL_WIDTH_M1: 472 st.ifm.width = param; 473 break; 474 case NPU_SET_KERNEL_HEIGHT_M1: 475 st.ifm.height[2] = param; 476 break; 477 case NPU_SET_KERNEL_STRIDE: 478 st.ifm.stride_kernel = param; 479 break; 480 case NPU_SET_IFM_PAD_TOP: 481 st.ifm.pad_top = param & 0x7f; 482 break; 483 case NPU_SET_IFM_PAD_LEFT: 484 st.ifm.pad_left = param & 0x7f; 485 break; 486 case NPU_SET_IFM_PAD_RIGHT: 487 st.ifm.pad_right = param & 0xff; 488 break; 489 case NPU_SET_IFM_PAD_BOTTOM: 490 st.ifm.pad_bottom = param & 0xff; 491 break; 492 case NPU_SET_IFM_DEPTH_M1: 493 st.ifm.depth = param; 494 break; 495 case NPU_SET_IFM_PRECISION: 496 st.ifm.precision = param; 497 break; 498 case NPU_SET_IFM_BROADCAST: 499 st.ifm.broadcast = param; 500 break; 501 case NPU_SET_IFM_REGION: 502 st.ifm.region = param & 0x7; 503 break; 504 case NPU_SET_IFM_WIDTH0_M1: 505 st.ifm.width0 = param; 506 break; 507 case NPU_SET_IFM_HEIGHT0_M1: 508 st.ifm.height[0] = param; 509 break; 510 case NPU_SET_IFM_HEIGHT1_M1: 511 st.ifm.height[1] = param; 512 break; 513 case NPU_SET_IFM_BASE0: 514 case NPU_SET_IFM_BASE1: 515 case NPU_SET_IFM_BASE2: 516 case NPU_SET_IFM_BASE3: 517 st.ifm.base[cmd & 0x3] = addr; 518 break; 519 case NPU_SET_IFM_STRIDE_X: 520 st.ifm.stride_x = addr; 521 break; 522 case NPU_SET_IFM_STRIDE_Y: 523 st.ifm.stride_y = addr; 524 break; 525 case NPU_SET_IFM_STRIDE_C: 526 st.ifm.stride_c = addr; 527 break; 528 529 case NPU_SET_OFM_WIDTH_M1: 530 st.ofm.width = param; 531 break; 532 case NPU_SET_OFM_HEIGHT_M1: 533 st.ofm.height[2] = param; 534 break; 535 case NPU_SET_OFM_DEPTH_M1: 536 st.ofm.depth = param; 537 break; 538 case NPU_SET_OFM_PRECISION: 539 st.ofm.precision = param; 540 break; 541 case NPU_SET_OFM_REGION: 542 st.ofm.region = param & 0x7; 543 break; 544 case NPU_SET_OFM_WIDTH0_M1: 545 st.ofm.width0 = param; 546 break; 547 case NPU_SET_OFM_HEIGHT0_M1: 548 st.ofm.height[0] = param; 549 break; 550 case NPU_SET_OFM_HEIGHT1_M1: 551 st.ofm.height[1] = param; 552 break; 553 case NPU_SET_OFM_BASE0: 554 case NPU_SET_OFM_BASE1: 555 case NPU_SET_OFM_BASE2: 556 case NPU_SET_OFM_BASE3: 557 st.ofm.base[cmd & 0x3] = addr; 558 break; 559 case NPU_SET_OFM_STRIDE_X: 560 st.ofm.stride_x = addr; 561 break; 562 case NPU_SET_OFM_STRIDE_Y: 563 st.ofm.stride_y = addr; 564 break; 565 case NPU_SET_OFM_STRIDE_C: 566 st.ofm.stride_c = addr; 567 break; 568 569 case NPU_SET_IFM2_BROADCAST: 570 st.ifm2.broadcast = param; 571 break; 572 case NPU_SET_IFM2_PRECISION: 573 st.ifm2.precision = param; 574 break; 575 case NPU_SET_IFM2_REGION: 576 st.ifm2.region = param & 0x7; 577 break; 578 case NPU_SET_IFM2_WIDTH0_M1: 579 st.ifm2.width0 = param; 580 break; 581 case NPU_SET_IFM2_HEIGHT0_M1: 582 st.ifm2.height[0] = param; 583 break; 584 case NPU_SET_IFM2_HEIGHT1_M1: 585 st.ifm2.height[1] = param; 586 break; 587 case NPU_SET_IFM2_BASE0: 588 case NPU_SET_IFM2_BASE1: 589 case NPU_SET_IFM2_BASE2: 590 case NPU_SET_IFM2_BASE3: 591 st.ifm2.base[cmd & 0x3] = addr; 592 break; 593 case NPU_SET_IFM2_STRIDE_X: 594 st.ifm2.stride_x = addr; 595 break; 596 case NPU_SET_IFM2_STRIDE_Y: 597 st.ifm2.stride_y = addr; 598 break; 599 case NPU_SET_IFM2_STRIDE_C: 600 st.ifm2.stride_c = addr; 601 break; 602 603 case NPU_SET_WEIGHT_REGION: 604 st.weight[0].region = param & 0x7; 605 break; 606 case NPU_SET_SCALE_REGION: 607 st.scale[0].region = param & 0x7; 608 break; 609 case NPU_SET_WEIGHT_BASE: 610 st.weight[0].base = addr; 611 break; 612 case NPU_SET_WEIGHT_LENGTH: 613 st.weight[0].length = cmds[1]; 614 break; 615 case NPU_SET_SCALE_BASE: 616 st.scale[0].base = addr; 617 break; 618 case NPU_SET_SCALE_LENGTH: 619 st.scale[0].length = cmds[1]; 620 break; 621 case NPU_SET_WEIGHT1_BASE: 622 st.weight[1].base = addr; 623 break; 624 case NPU_SET_WEIGHT1_LENGTH: 625 st.weight[1].length = cmds[1]; 626 break; 627 case NPU_SET_SCALE1_BASE: // NPU_SET_WEIGHT2_BASE (U85) 628 if (ethosu_is_u65(edev)) 629 st.scale[1].base = addr; 630 else 631 st.weight[2].base = addr; 632 break; 633 case NPU_SET_SCALE1_LENGTH: // NPU_SET_WEIGHT2_LENGTH (U85) 634 if (ethosu_is_u65(edev)) 635 st.scale[1].length = cmds[1]; 636 else 637 st.weight[2].length = cmds[1]; 638 break; 639 case NPU_SET_WEIGHT3_BASE: 640 st.weight[3].base = addr; 641 break; 642 case NPU_SET_WEIGHT3_LENGTH: 643 st.weight[3].length = cmds[1]; 644 break; 645 646 case NPU_SET_DMA0_SRC_REGION: 647 if (param & 0x100) 648 st.dma.src.region = -1; 649 else 650 st.dma.src.region = param & 0x7; 651 st.dma.mode = (param >> 9) & 0x3; 652 break; 653 case NPU_SET_DMA0_DST_REGION: 654 if (param & 0x100) 655 st.dma.dst.region = -1; 656 else 657 st.dma.dst.region = param & 0x7; 658 break; 659 case NPU_SET_DMA0_SIZE0: 660 st.dma.size0 = param; 661 break; 662 case NPU_SET_DMA0_SIZE1: 663 st.dma.size1 = param; 664 break; 665 case NPU_SET_DMA0_SRC_STRIDE0: 666 st.dma.src.stride[0] = ((s64)addr << 24) >> 24; 667 break; 668 case NPU_SET_DMA0_SRC_STRIDE1: 669 st.dma.src.stride[1] = ((s64)addr << 24) >> 24; 670 break; 671 case NPU_SET_DMA0_DST_STRIDE0: 672 st.dma.dst.stride[0] = ((s64)addr << 24) >> 24; 673 break; 674 case NPU_SET_DMA0_DST_STRIDE1: 675 st.dma.dst.stride[1] = ((s64)addr << 24) >> 24; 676 break; 677 case NPU_SET_DMA0_SRC: 678 st.dma.src.offset = addr; 679 break; 680 case NPU_SET_DMA0_DST: 681 st.dma.dst.offset = addr; 682 break; 683 case NPU_SET_DMA0_LEN: 684 st.dma.src.len = st.dma.dst.len = addr; 685 break; 686 default: 687 break; 688 } 689 } 690 691 for (i = 0; i < NPU_BASEP_REGION_MAX; i++) { 692 if (!info->region_size[i]) 693 continue; 694 dev_dbg(ddev->dev, "region %d max size: 0x%llx\n", 695 i, info->region_size[i]); 696 } 697 698 bo->info = no_free_ptr(info); 699 return 0; 700 } 701 702 /** 703 * ethosu_gem_cmdstream_create() - Create a GEM object and attach it to a handle. 704 * @file: DRM file. 705 * @ddev: DRM device. 706 * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared. 707 * @size: Size of the GEM object to allocate. 708 * @flags: Combination of drm_ethosu_bo_flags flags. 709 * @handle: Pointer holding the handle pointing to the new GEM object. 710 * 711 * Return: Zero on success 712 */ 713 int ethosu_gem_cmdstream_create(struct drm_file *file, 714 struct drm_device *ddev, 715 u32 size, u64 data, u32 flags, u32 *handle) 716 { 717 int ret; 718 struct drm_gem_dma_object *mem; 719 struct ethosu_gem_object *bo; 720 721 mem = drm_gem_dma_create(ddev, size); 722 if (IS_ERR(mem)) 723 return PTR_ERR(mem); 724 725 bo = to_ethosu_bo(&mem->base); 726 bo->flags = flags; 727 728 ret = ethosu_gem_cmdstream_copy_and_validate(ddev, 729 (void __user *)(uintptr_t)data, 730 bo, size); 731 if (ret) 732 goto fail; 733 734 /* 735 * Allocate an id of idr table where the obj is registered 736 * and handle has the id what user can see. 737 */ 738 ret = drm_gem_handle_create(file, &mem->base, handle); 739 740 fail: 741 /* drop reference from allocate - handle holds it now. */ 742 drm_gem_object_put(&mem->base); 743 744 return ret; 745 } 746