1 // SPDX-License-Identifier: GPL-2.0-only or MIT 2 /* Copyright 2025 Arm, Ltd. */ 3 4 #include <linux/err.h> 5 #include <linux/slab.h> 6 7 #include <drm/ethosu_accel.h> 8 9 #include "ethosu_device.h" 10 #include "ethosu_gem.h" 11 12 static void ethosu_gem_free_object(struct drm_gem_object *obj) 13 { 14 struct ethosu_gem_object *bo = to_ethosu_bo(obj); 15 16 kfree(bo->info); 17 drm_gem_free_mmap_offset(&bo->base.base); 18 drm_gem_dma_free(&bo->base); 19 } 20 21 static int ethosu_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 22 { 23 struct ethosu_gem_object *bo = to_ethosu_bo(obj); 24 25 /* Don't allow mmap on objects that have the NO_MMAP flag set. */ 26 if (bo->flags & DRM_ETHOSU_BO_NO_MMAP) 27 return -EINVAL; 28 29 return drm_gem_dma_object_mmap(obj, vma); 30 } 31 32 static const struct drm_gem_object_funcs ethosu_gem_funcs = { 33 .free = ethosu_gem_free_object, 34 .print_info = drm_gem_dma_object_print_info, 35 .get_sg_table = drm_gem_dma_object_get_sg_table, 36 .vmap = drm_gem_dma_object_vmap, 37 .mmap = ethosu_gem_mmap, 38 .vm_ops = &drm_gem_dma_vm_ops, 39 }; 40 41 /** 42 * ethosu_gem_create_object - Implementation of driver->gem_create_object. 43 * @ddev: DRM device 44 * @size: Size in bytes of the memory the object will reference 45 * 46 * This lets the GEM helpers allocate object structs for us, and keep 47 * our BO stats correct. 48 */ 49 struct drm_gem_object *ethosu_gem_create_object(struct drm_device *ddev, size_t size) 50 { 51 struct ethosu_gem_object *obj; 52 53 obj = kzalloc(sizeof(*obj), GFP_KERNEL); 54 if (!obj) 55 return ERR_PTR(-ENOMEM); 56 57 obj->base.base.funcs = ðosu_gem_funcs; 58 return &obj->base.base; 59 } 60 61 /** 62 * ethosu_gem_create_with_handle() - Create a GEM object and attach it to a handle. 63 * @file: DRM file. 64 * @ddev: DRM device. 65 * @size: Size of the GEM object to allocate. 66 * @flags: Combination of drm_ethosu_bo_flags flags. 67 * @handle: Pointer holding the handle pointing to the new GEM object. 68 * 69 * Return: Zero on success 70 */ 71 int ethosu_gem_create_with_handle(struct drm_file *file, 72 struct drm_device *ddev, 73 u64 *size, u32 flags, u32 *handle) 74 { 75 struct drm_gem_dma_object *mem; 76 struct ethosu_gem_object *bo; 77 int ret; 78 79 mem = drm_gem_dma_create(ddev, *size); 80 if (IS_ERR(mem)) 81 return PTR_ERR(mem); 82 83 bo = to_ethosu_bo(&mem->base); 84 bo->flags = flags; 85 86 /* 87 * Allocate an id of idr table where the obj is registered 88 * and handle has the id what user can see. 89 */ 90 ret = drm_gem_handle_create(file, &mem->base, handle); 91 if (!ret) 92 *size = bo->base.base.size; 93 94 /* drop reference from allocate - handle holds it now. */ 95 drm_gem_object_put(&mem->base); 96 97 return ret; 98 } 99 100 struct dma { 101 s8 region; 102 u64 len; 103 u64 offset; 104 s64 stride[2]; 105 }; 106 107 struct dma_state { 108 u16 size0; 109 u16 size1; 110 s8 mode; 111 struct dma src; 112 struct dma dst; 113 }; 114 115 struct buffer { 116 u64 base; 117 u32 length; 118 s8 region; 119 }; 120 121 struct feat_matrix { 122 u64 base[4]; 123 s64 stride_x; 124 s64 stride_y; 125 s64 stride_c; 126 s8 region; 127 u8 broadcast; 128 u16 stride_kernel; 129 u16 precision; 130 u16 depth; 131 u16 width; 132 u16 width0; 133 u16 height[3]; 134 u8 pad_top; 135 u8 pad_left; 136 u8 pad_bottom; 137 u8 pad_right; 138 }; 139 140 struct cmd_state { 141 struct dma_state dma; 142 struct buffer scale[2]; 143 struct buffer weight[4]; 144 struct feat_matrix ofm; 145 struct feat_matrix ifm; 146 struct feat_matrix ifm2; 147 }; 148 149 static void cmd_state_init(struct cmd_state *st) 150 { 151 /* Initialize to all 1s to detect missing setup */ 152 memset(st, 0xff, sizeof(*st)); 153 } 154 155 static u64 cmd_to_addr(u32 *cmd) 156 { 157 return ((u64)((cmd[0] & 0xff0000) << 16)) | cmd[1]; 158 } 159 160 static u64 dma_length(struct ethosu_validated_cmdstream_info *info, 161 struct dma_state *dma_st, struct dma *dma) 162 { 163 s8 mode = dma_st->mode; 164 u64 len = dma->len; 165 166 if (mode >= 1) { 167 len += dma->stride[0]; 168 len *= dma_st->size0; 169 } 170 if (mode == 2) { 171 len += dma->stride[1]; 172 len *= dma_st->size1; 173 } 174 if (dma->region >= 0) 175 info->region_size[dma->region] = max(info->region_size[dma->region], 176 len + dma->offset); 177 178 return len; 179 } 180 181 static u64 feat_matrix_length(struct ethosu_validated_cmdstream_info *info, 182 struct feat_matrix *fm, 183 u32 x, u32 y, u32 c) 184 { 185 u32 element_size, storage = fm->precision >> 14; 186 int tile = 0; 187 u64 addr; 188 189 if (fm->region < 0) 190 return U64_MAX; 191 192 switch (storage) { 193 case 0: 194 if (x >= fm->width0 + 1) { 195 x -= fm->width0 + 1; 196 tile += 1; 197 } 198 if (y >= fm->height[tile] + 1) { 199 y -= fm->height[tile] + 1; 200 tile += 2; 201 } 202 break; 203 case 1: 204 if (y >= fm->height[1] + 1) { 205 y -= fm->height[1] + 1; 206 tile = 2; 207 } else if (y >= fm->height[0] + 1) { 208 y -= fm->height[0] + 1; 209 tile = 1; 210 } 211 break; 212 } 213 if (fm->base[tile] == U64_MAX) 214 return U64_MAX; 215 216 addr = fm->base[tile] + y * fm->stride_y; 217 218 switch ((fm->precision >> 6) & 0x3) { // format 219 case 0: //nhwc: 220 addr += x * fm->stride_x + c; 221 break; 222 case 1: //nhcwb16: 223 element_size = BIT((fm->precision >> 1) & 0x3); 224 225 addr += (c / 16) * fm->stride_c + (16 * x + (c & 0xf)) * element_size; 226 break; 227 } 228 229 info->region_size[fm->region] = max(info->region_size[fm->region], addr + 1); 230 231 return addr; 232 } 233 234 static int calc_sizes(struct drm_device *ddev, 235 struct ethosu_validated_cmdstream_info *info, 236 u16 op, struct cmd_state *st, 237 bool ifm, bool ifm2, bool weight, bool scale) 238 { 239 u64 len; 240 241 if (ifm) { 242 if (st->ifm.stride_kernel == U16_MAX) 243 return -EINVAL; 244 u32 stride_y = ((st->ifm.stride_kernel >> 8) & 0x2) + 245 ((st->ifm.stride_kernel >> 1) & 0x1) + 1; 246 u32 stride_x = ((st->ifm.stride_kernel >> 5) & 0x2) + 247 (st->ifm.stride_kernel & 0x1) + 1; 248 u32 ifm_height = st->ofm.height[2] * stride_y + 249 st->ifm.height[2] - (st->ifm.pad_top + st->ifm.pad_bottom); 250 u32 ifm_width = st->ofm.width * stride_x + 251 st->ifm.width - (st->ifm.pad_left + st->ifm.pad_right); 252 253 len = feat_matrix_length(info, &st->ifm, ifm_width, 254 ifm_height, st->ifm.depth); 255 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n", 256 op, st->ifm.region, st->ifm.base[0], len); 257 if (len == U64_MAX) 258 return -EINVAL; 259 } 260 261 if (ifm2) { 262 len = feat_matrix_length(info, &st->ifm2, st->ifm.depth, 263 0, st->ofm.depth); 264 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n", 265 op, st->ifm2.region, st->ifm2.base[0], len); 266 if (len == U64_MAX) 267 return -EINVAL; 268 } 269 270 if (weight) { 271 dev_dbg(ddev->dev, "op %d: W:%d:0x%llx-0x%llx\n", 272 op, st->weight[0].region, st->weight[0].base, 273 st->weight[0].base + st->weight[0].length - 1); 274 if (st->weight[0].region < 0 || st->weight[0].base == U64_MAX || 275 st->weight[0].length == U32_MAX) 276 return -EINVAL; 277 info->region_size[st->weight[0].region] = 278 max(info->region_size[st->weight[0].region], 279 st->weight[0].base + st->weight[0].length); 280 } 281 282 if (scale) { 283 dev_dbg(ddev->dev, "op %d: S:%d:0x%llx-0x%llx\n", 284 op, st->scale[0].region, st->scale[0].base, 285 st->scale[0].base + st->scale[0].length - 1); 286 if (st->scale[0].region < 0 || st->scale[0].base == U64_MAX || 287 st->scale[0].length == U32_MAX) 288 return -EINVAL; 289 info->region_size[st->scale[0].region] = 290 max(info->region_size[st->scale[0].region], 291 st->scale[0].base + st->scale[0].length); 292 } 293 294 len = feat_matrix_length(info, &st->ofm, st->ofm.width, 295 st->ofm.height[2], st->ofm.depth); 296 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n", 297 op, st->ofm.region, st->ofm.base[0], len); 298 if (len == U64_MAX) 299 return -EINVAL; 300 info->output_region[st->ofm.region] = true; 301 302 return 0; 303 } 304 305 static int calc_sizes_elemwise(struct drm_device *ddev, 306 struct ethosu_validated_cmdstream_info *info, 307 u16 op, struct cmd_state *st, 308 bool ifm, bool ifm2) 309 { 310 u32 height, width, depth; 311 u64 len; 312 313 if (ifm) { 314 height = st->ifm.broadcast & 0x1 ? 0 : st->ofm.height[2]; 315 width = st->ifm.broadcast & 0x2 ? 0 : st->ofm.width; 316 depth = st->ifm.broadcast & 0x4 ? 0 : st->ofm.depth; 317 318 len = feat_matrix_length(info, &st->ifm, width, 319 height, depth); 320 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n", 321 op, st->ifm.region, st->ifm.base[0], len); 322 if (len == U64_MAX) 323 return -EINVAL; 324 } 325 326 if (ifm2) { 327 height = st->ifm2.broadcast & 0x1 ? 0 : st->ofm.height[2]; 328 width = st->ifm2.broadcast & 0x2 ? 0 : st->ofm.width; 329 depth = st->ifm2.broadcast & 0x4 ? 0 : st->ofm.depth; 330 331 len = feat_matrix_length(info, &st->ifm2, width, 332 height, depth); 333 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n", 334 op, st->ifm2.region, st->ifm2.base[0], len); 335 if (len == U64_MAX) 336 return -EINVAL; 337 } 338 339 len = feat_matrix_length(info, &st->ofm, st->ofm.width, 340 st->ofm.height[2], st->ofm.depth); 341 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n", 342 op, st->ofm.region, st->ofm.base[0], len); 343 if (len == U64_MAX) 344 return -EINVAL; 345 info->output_region[st->ofm.region] = true; 346 347 return 0; 348 } 349 350 static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev, 351 u32 __user *ucmds, 352 struct ethosu_gem_object *bo, 353 u32 size) 354 { 355 struct ethosu_validated_cmdstream_info __free(kfree) *info = kzalloc(sizeof(*info), GFP_KERNEL); 356 struct ethosu_device *edev = to_ethosu_device(ddev); 357 u32 *bocmds = bo->base.vaddr; 358 struct cmd_state st; 359 int i, ret; 360 361 if (!info) 362 return -ENOMEM; 363 info->cmd_size = size; 364 365 cmd_state_init(&st); 366 367 for (i = 0; i < size / 4; i++) { 368 bool use_ifm, use_ifm2, use_scale; 369 u64 dstlen, srclen; 370 u16 cmd, param; 371 u32 cmds[2]; 372 u64 addr; 373 374 if (get_user(cmds[0], ucmds++)) 375 return -EFAULT; 376 377 bocmds[i] = cmds[0]; 378 379 cmd = cmds[0]; 380 param = cmds[0] >> 16; 381 382 if (cmd & 0x4000) { 383 if (get_user(cmds[1], ucmds++)) 384 return -EFAULT; 385 386 i++; 387 bocmds[i] = cmds[1]; 388 addr = cmd_to_addr(cmds); 389 } 390 391 switch (cmd) { 392 case NPU_OP_DMA_START: 393 srclen = dma_length(info, &st.dma, &st.dma.src); 394 dstlen = dma_length(info, &st.dma, &st.dma.dst); 395 396 if (st.dma.dst.region >= 0) 397 info->output_region[st.dma.dst.region] = true; 398 dev_dbg(ddev->dev, "cmd: DMA SRC:%d:0x%llx+0x%llx DST:%d:0x%llx+0x%llx\n", 399 st.dma.src.region, st.dma.src.offset, srclen, 400 st.dma.dst.region, st.dma.dst.offset, dstlen); 401 break; 402 case NPU_OP_CONV: 403 case NPU_OP_DEPTHWISE: 404 use_ifm2 = param & 0x1; // weights_ifm2 405 use_scale = !(st.ofm.precision & 0x100); 406 ret = calc_sizes(ddev, info, cmd, &st, true, use_ifm2, 407 !use_ifm2, use_scale); 408 if (ret) 409 return ret; 410 break; 411 case NPU_OP_POOL: 412 use_ifm = param != 0x4; // pooling mode 413 use_scale = !(st.ofm.precision & 0x100); 414 ret = calc_sizes(ddev, info, cmd, &st, use_ifm, false, 415 false, use_scale); 416 if (ret) 417 return ret; 418 break; 419 case NPU_OP_ELEMENTWISE: 420 use_ifm2 = !((st.ifm2.broadcast == 8) || (param == 5) || 421 (param == 6) || (param == 7) || (param == 0x24)); 422 use_ifm = st.ifm.broadcast != 8; 423 ret = calc_sizes_elemwise(ddev, info, cmd, &st, use_ifm, use_ifm2); 424 if (ret) 425 return ret; 426 break; 427 case NPU_OP_RESIZE: // U85 only 428 WARN_ON(1); // TODO 429 break; 430 case NPU_SET_KERNEL_WIDTH_M1: 431 st.ifm.width = param; 432 break; 433 case NPU_SET_KERNEL_HEIGHT_M1: 434 st.ifm.height[2] = param; 435 break; 436 case NPU_SET_KERNEL_STRIDE: 437 st.ifm.stride_kernel = param; 438 break; 439 case NPU_SET_IFM_PAD_TOP: 440 st.ifm.pad_top = param & 0x7f; 441 break; 442 case NPU_SET_IFM_PAD_LEFT: 443 st.ifm.pad_left = param & 0x7f; 444 break; 445 case NPU_SET_IFM_PAD_RIGHT: 446 st.ifm.pad_right = param & 0xff; 447 break; 448 case NPU_SET_IFM_PAD_BOTTOM: 449 st.ifm.pad_bottom = param & 0xff; 450 break; 451 case NPU_SET_IFM_DEPTH_M1: 452 st.ifm.depth = param; 453 break; 454 case NPU_SET_IFM_PRECISION: 455 st.ifm.precision = param; 456 break; 457 case NPU_SET_IFM_BROADCAST: 458 st.ifm.broadcast = param; 459 break; 460 case NPU_SET_IFM_REGION: 461 st.ifm.region = param & 0x7f; 462 break; 463 case NPU_SET_IFM_WIDTH0_M1: 464 st.ifm.width0 = param; 465 break; 466 case NPU_SET_IFM_HEIGHT0_M1: 467 st.ifm.height[0] = param; 468 break; 469 case NPU_SET_IFM_HEIGHT1_M1: 470 st.ifm.height[1] = param; 471 break; 472 case NPU_SET_IFM_BASE0: 473 case NPU_SET_IFM_BASE1: 474 case NPU_SET_IFM_BASE2: 475 case NPU_SET_IFM_BASE3: 476 st.ifm.base[cmd & 0x3] = addr; 477 break; 478 case NPU_SET_IFM_STRIDE_X: 479 st.ifm.stride_x = addr; 480 break; 481 case NPU_SET_IFM_STRIDE_Y: 482 st.ifm.stride_y = addr; 483 break; 484 case NPU_SET_IFM_STRIDE_C: 485 st.ifm.stride_c = addr; 486 break; 487 488 case NPU_SET_OFM_WIDTH_M1: 489 st.ofm.width = param; 490 break; 491 case NPU_SET_OFM_HEIGHT_M1: 492 st.ofm.height[2] = param; 493 break; 494 case NPU_SET_OFM_DEPTH_M1: 495 st.ofm.depth = param; 496 break; 497 case NPU_SET_OFM_PRECISION: 498 st.ofm.precision = param; 499 break; 500 case NPU_SET_OFM_REGION: 501 st.ofm.region = param & 0x7; 502 break; 503 case NPU_SET_OFM_WIDTH0_M1: 504 st.ofm.width0 = param; 505 break; 506 case NPU_SET_OFM_HEIGHT0_M1: 507 st.ofm.height[0] = param; 508 break; 509 case NPU_SET_OFM_HEIGHT1_M1: 510 st.ofm.height[1] = param; 511 break; 512 case NPU_SET_OFM_BASE0: 513 case NPU_SET_OFM_BASE1: 514 case NPU_SET_OFM_BASE2: 515 case NPU_SET_OFM_BASE3: 516 st.ofm.base[cmd & 0x3] = addr; 517 break; 518 case NPU_SET_OFM_STRIDE_X: 519 st.ofm.stride_x = addr; 520 break; 521 case NPU_SET_OFM_STRIDE_Y: 522 st.ofm.stride_y = addr; 523 break; 524 case NPU_SET_OFM_STRIDE_C: 525 st.ofm.stride_c = addr; 526 break; 527 528 case NPU_SET_IFM2_BROADCAST: 529 st.ifm2.broadcast = param; 530 break; 531 case NPU_SET_IFM2_PRECISION: 532 st.ifm2.precision = param; 533 break; 534 case NPU_SET_IFM2_REGION: 535 st.ifm2.region = param & 0x7; 536 break; 537 case NPU_SET_IFM2_WIDTH0_M1: 538 st.ifm2.width0 = param; 539 break; 540 case NPU_SET_IFM2_HEIGHT0_M1: 541 st.ifm2.height[0] = param; 542 break; 543 case NPU_SET_IFM2_HEIGHT1_M1: 544 st.ifm2.height[1] = param; 545 break; 546 case NPU_SET_IFM2_BASE0: 547 case NPU_SET_IFM2_BASE1: 548 case NPU_SET_IFM2_BASE2: 549 case NPU_SET_IFM2_BASE3: 550 st.ifm2.base[cmd & 0x3] = addr; 551 break; 552 case NPU_SET_IFM2_STRIDE_X: 553 st.ifm2.stride_x = addr; 554 break; 555 case NPU_SET_IFM2_STRIDE_Y: 556 st.ifm2.stride_y = addr; 557 break; 558 case NPU_SET_IFM2_STRIDE_C: 559 st.ifm2.stride_c = addr; 560 break; 561 562 case NPU_SET_WEIGHT_REGION: 563 st.weight[0].region = param & 0x7; 564 break; 565 case NPU_SET_SCALE_REGION: 566 st.scale[0].region = param & 0x7; 567 break; 568 case NPU_SET_WEIGHT_BASE: 569 st.weight[0].base = addr; 570 break; 571 case NPU_SET_WEIGHT_LENGTH: 572 st.weight[0].length = cmds[1]; 573 break; 574 case NPU_SET_SCALE_BASE: 575 st.scale[0].base = addr; 576 break; 577 case NPU_SET_SCALE_LENGTH: 578 st.scale[0].length = cmds[1]; 579 break; 580 case NPU_SET_WEIGHT1_BASE: 581 st.weight[1].base = addr; 582 break; 583 case NPU_SET_WEIGHT1_LENGTH: 584 st.weight[1].length = cmds[1]; 585 break; 586 case NPU_SET_SCALE1_BASE: // NPU_SET_WEIGHT2_BASE (U85) 587 if (ethosu_is_u65(edev)) 588 st.scale[1].base = addr; 589 else 590 st.weight[2].base = addr; 591 break; 592 case NPU_SET_SCALE1_LENGTH: // NPU_SET_WEIGHT2_LENGTH (U85) 593 if (ethosu_is_u65(edev)) 594 st.scale[1].length = cmds[1]; 595 else 596 st.weight[1].length = cmds[1]; 597 break; 598 case NPU_SET_WEIGHT3_BASE: 599 st.weight[3].base = addr; 600 break; 601 case NPU_SET_WEIGHT3_LENGTH: 602 st.weight[3].length = cmds[1]; 603 break; 604 605 case NPU_SET_DMA0_SRC_REGION: 606 if (param & 0x100) 607 st.dma.src.region = -1; 608 else 609 st.dma.src.region = param & 0x7; 610 st.dma.mode = (param >> 9) & 0x3; 611 break; 612 case NPU_SET_DMA0_DST_REGION: 613 if (param & 0x100) 614 st.dma.dst.region = -1; 615 else 616 st.dma.dst.region = param & 0x7; 617 break; 618 case NPU_SET_DMA0_SIZE0: 619 st.dma.size0 = param; 620 break; 621 case NPU_SET_DMA0_SIZE1: 622 st.dma.size1 = param; 623 break; 624 case NPU_SET_DMA0_SRC_STRIDE0: 625 st.dma.src.stride[0] = ((s64)addr << 24) >> 24; 626 break; 627 case NPU_SET_DMA0_SRC_STRIDE1: 628 st.dma.src.stride[1] = ((s64)addr << 24) >> 24; 629 break; 630 case NPU_SET_DMA0_DST_STRIDE0: 631 st.dma.dst.stride[0] = ((s64)addr << 24) >> 24; 632 break; 633 case NPU_SET_DMA0_DST_STRIDE1: 634 st.dma.dst.stride[1] = ((s64)addr << 24) >> 24; 635 break; 636 case NPU_SET_DMA0_SRC: 637 st.dma.src.offset = addr; 638 break; 639 case NPU_SET_DMA0_DST: 640 st.dma.dst.offset = addr; 641 break; 642 case NPU_SET_DMA0_LEN: 643 st.dma.src.len = st.dma.dst.len = addr; 644 break; 645 default: 646 break; 647 } 648 } 649 650 for (i = 0; i < NPU_BASEP_REGION_MAX; i++) { 651 if (!info->region_size[i]) 652 continue; 653 dev_dbg(ddev->dev, "region %d max size: 0x%llx\n", 654 i, info->region_size[i]); 655 } 656 657 bo->info = no_free_ptr(info); 658 return 0; 659 } 660 661 /** 662 * ethosu_gem_cmdstream_create() - Create a GEM object and attach it to a handle. 663 * @file: DRM file. 664 * @ddev: DRM device. 665 * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared. 666 * @size: Size of the GEM object to allocate. 667 * @flags: Combination of drm_ethosu_bo_flags flags. 668 * @handle: Pointer holding the handle pointing to the new GEM object. 669 * 670 * Return: Zero on success 671 */ 672 int ethosu_gem_cmdstream_create(struct drm_file *file, 673 struct drm_device *ddev, 674 u32 size, u64 data, u32 flags, u32 *handle) 675 { 676 int ret; 677 struct drm_gem_dma_object *mem; 678 struct ethosu_gem_object *bo; 679 680 mem = drm_gem_dma_create(ddev, size); 681 if (IS_ERR(mem)) 682 return PTR_ERR(mem); 683 684 bo = to_ethosu_bo(&mem->base); 685 bo->flags = flags; 686 687 ret = ethosu_gem_cmdstream_copy_and_validate(ddev, 688 (void __user *)(uintptr_t)data, 689 bo, size); 690 if (ret) 691 goto fail; 692 693 /* 694 * Allocate an id of idr table where the obj is registered 695 * and handle has the id what user can see. 696 */ 697 ret = drm_gem_handle_create(file, &mem->base, handle); 698 699 fail: 700 /* drop reference from allocate - handle holds it now. */ 701 drm_gem_object_put(&mem->base); 702 703 return ret; 704 } 705