1 // SPDX-License-Identifier: GPL-2.0-only or MIT 2 /* Copyright 2025 Arm, Ltd. */ 3 4 #include <linux/err.h> 5 #include <linux/slab.h> 6 7 #include <drm/ethosu_accel.h> 8 9 #include "ethosu_device.h" 10 #include "ethosu_gem.h" 11 12 static void ethosu_gem_free_object(struct drm_gem_object *obj) 13 { 14 struct ethosu_gem_object *bo = to_ethosu_bo(obj); 15 16 kfree(bo->info); 17 drm_gem_free_mmap_offset(&bo->base.base); 18 drm_gem_dma_free(&bo->base); 19 } 20 21 static int ethosu_gem_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma) 22 { 23 struct ethosu_gem_object *bo = to_ethosu_bo(obj); 24 25 /* Don't allow mmap on objects that have the NO_MMAP flag set. */ 26 if (bo->flags & DRM_ETHOSU_BO_NO_MMAP) 27 return -EINVAL; 28 29 return drm_gem_dma_object_mmap(obj, vma); 30 } 31 32 static const struct drm_gem_object_funcs ethosu_gem_funcs = { 33 .free = ethosu_gem_free_object, 34 .print_info = drm_gem_dma_object_print_info, 35 .get_sg_table = drm_gem_dma_object_get_sg_table, 36 .vmap = drm_gem_dma_object_vmap, 37 .mmap = ethosu_gem_mmap, 38 .vm_ops = &drm_gem_dma_vm_ops, 39 }; 40 41 /** 42 * ethosu_gem_create_object - Implementation of driver->gem_create_object. 43 * @ddev: DRM device 44 * @size: Size in bytes of the memory the object will reference 45 * 46 * This lets the GEM helpers allocate object structs for us, and keep 47 * our BO stats correct. 48 */ 49 struct drm_gem_object *ethosu_gem_create_object(struct drm_device *ddev, size_t size) 50 { 51 struct ethosu_gem_object *obj; 52 53 obj = kzalloc_obj(*obj); 54 if (!obj) 55 return ERR_PTR(-ENOMEM); 56 57 obj->base.base.funcs = ðosu_gem_funcs; 58 return &obj->base.base; 59 } 60 61 /** 62 * ethosu_gem_create_with_handle() - Create a GEM object and attach it to a handle. 63 * @file: DRM file. 64 * @ddev: DRM device. 65 * @size: Size of the GEM object to allocate. 66 * @flags: Combination of drm_ethosu_bo_flags flags. 67 * @handle: Pointer holding the handle pointing to the new GEM object. 68 * 69 * Return: Zero on success 70 */ 71 int ethosu_gem_create_with_handle(struct drm_file *file, 72 struct drm_device *ddev, 73 u64 *size, u32 flags, u32 *handle) 74 { 75 struct drm_gem_dma_object *mem; 76 struct ethosu_gem_object *bo; 77 int ret; 78 79 mem = drm_gem_dma_create(ddev, *size); 80 if (IS_ERR(mem)) 81 return PTR_ERR(mem); 82 83 bo = to_ethosu_bo(&mem->base); 84 bo->flags = flags; 85 86 /* 87 * Allocate an id of idr table where the obj is registered 88 * and handle has the id what user can see. 89 */ 90 ret = drm_gem_handle_create(file, &mem->base, handle); 91 if (!ret) 92 *size = bo->base.base.size; 93 94 /* drop reference from allocate - handle holds it now. */ 95 drm_gem_object_put(&mem->base); 96 97 return ret; 98 } 99 100 struct dma { 101 s8 region; 102 u64 len; 103 u64 offset; 104 s64 stride[2]; 105 }; 106 107 struct dma_state { 108 u16 size0; 109 u16 size1; 110 s8 mode; 111 struct dma src; 112 struct dma dst; 113 }; 114 115 struct buffer { 116 u64 base; 117 u32 length; 118 s8 region; 119 }; 120 121 struct feat_matrix { 122 u64 base[4]; 123 s64 stride_x; 124 s64 stride_y; 125 s64 stride_c; 126 s8 region; 127 u8 broadcast; 128 u16 stride_kernel; 129 u16 precision; 130 u16 depth; 131 u16 width; 132 u16 width0; 133 u16 height[3]; 134 u8 pad_top; 135 u8 pad_left; 136 u8 pad_bottom; 137 u8 pad_right; 138 }; 139 140 struct cmd_state { 141 struct dma_state dma; 142 struct buffer scale[2]; 143 struct buffer weight[4]; 144 struct feat_matrix ofm; 145 struct feat_matrix ifm; 146 struct feat_matrix ifm2; 147 }; 148 149 static void cmd_state_init(struct cmd_state *st) 150 { 151 /* Initialize to all 1s to detect missing setup */ 152 memset(st, 0xff, sizeof(*st)); 153 } 154 155 static u64 cmd_to_addr(u32 *cmd) 156 { 157 return ((u64)((cmd[0] & 0xff0000) << 16)) | cmd[1]; 158 } 159 160 static u64 dma_length(struct ethosu_validated_cmdstream_info *info, 161 struct dma_state *dma_st, struct dma *dma) 162 { 163 s8 mode = dma_st->mode; 164 u64 len = dma->len; 165 166 if (mode >= 1) { 167 len += dma->stride[0]; 168 len *= dma_st->size0; 169 } 170 if (mode == 2) { 171 len += dma->stride[1]; 172 len *= dma_st->size1; 173 } 174 if (dma->region >= 0) 175 info->region_size[dma->region] = max(info->region_size[dma->region], 176 len + dma->offset); 177 178 return len; 179 } 180 181 static u64 feat_matrix_length(struct ethosu_validated_cmdstream_info *info, 182 struct feat_matrix *fm, 183 u32 x, u32 y, u32 c) 184 { 185 u32 element_size, storage = fm->precision >> 14; 186 int tile = 0; 187 u64 addr; 188 189 if (fm->region < 0) 190 return U64_MAX; 191 192 switch (storage) { 193 case 0: 194 if (x >= fm->width0 + 1) { 195 x -= fm->width0 + 1; 196 tile += 1; 197 } 198 if (y >= fm->height[tile] + 1) { 199 y -= fm->height[tile] + 1; 200 tile += 2; 201 } 202 break; 203 case 1: 204 if (y >= fm->height[1] + 1) { 205 y -= fm->height[1] + 1; 206 tile = 2; 207 } else if (y >= fm->height[0] + 1) { 208 y -= fm->height[0] + 1; 209 tile = 1; 210 } 211 break; 212 } 213 if (fm->base[tile] == U64_MAX) 214 return U64_MAX; 215 216 addr = fm->base[tile] + y * fm->stride_y; 217 218 switch ((fm->precision >> 6) & 0x3) { // format 219 case 0: //nhwc: 220 addr += x * fm->stride_x + c; 221 break; 222 case 1: //nhcwb16: 223 element_size = BIT((fm->precision >> 1) & 0x3); 224 225 addr += (c / 16) * fm->stride_c + (16 * x + (c & 0xf)) * element_size; 226 break; 227 } 228 229 info->region_size[fm->region] = max(info->region_size[fm->region], addr + 1); 230 231 return addr; 232 } 233 234 static int calc_sizes(struct drm_device *ddev, 235 struct ethosu_validated_cmdstream_info *info, 236 u16 op, struct cmd_state *st, 237 bool ifm, bool ifm2, bool weight, bool scale) 238 { 239 u64 len; 240 241 if (ifm) { 242 if (st->ifm.stride_kernel == U16_MAX) 243 return -EINVAL; 244 u32 stride_y = ((st->ifm.stride_kernel >> 8) & 0x2) + 245 ((st->ifm.stride_kernel >> 1) & 0x1) + 1; 246 u32 stride_x = ((st->ifm.stride_kernel >> 5) & 0x2) + 247 (st->ifm.stride_kernel & 0x1) + 1; 248 u32 ifm_height = st->ofm.height[2] * stride_y + 249 st->ifm.height[2] - (st->ifm.pad_top + st->ifm.pad_bottom); 250 u32 ifm_width = st->ofm.width * stride_x + 251 st->ifm.width - (st->ifm.pad_left + st->ifm.pad_right); 252 253 len = feat_matrix_length(info, &st->ifm, ifm_width, 254 ifm_height, st->ifm.depth); 255 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n", 256 op, st->ifm.region, st->ifm.base[0], len); 257 if (len == U64_MAX) 258 return -EINVAL; 259 } 260 261 if (ifm2) { 262 len = feat_matrix_length(info, &st->ifm2, st->ifm.depth, 263 0, st->ofm.depth); 264 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n", 265 op, st->ifm2.region, st->ifm2.base[0], len); 266 if (len == U64_MAX) 267 return -EINVAL; 268 } 269 270 if (weight) { 271 dev_dbg(ddev->dev, "op %d: W:%d:0x%llx-0x%llx\n", 272 op, st->weight[0].region, st->weight[0].base, 273 st->weight[0].base + st->weight[0].length - 1); 274 if (st->weight[0].region < 0 || st->weight[0].base == U64_MAX || 275 st->weight[0].length == U32_MAX) 276 return -EINVAL; 277 info->region_size[st->weight[0].region] = 278 max(info->region_size[st->weight[0].region], 279 st->weight[0].base + st->weight[0].length); 280 } 281 282 if (scale) { 283 dev_dbg(ddev->dev, "op %d: S:%d:0x%llx-0x%llx\n", 284 op, st->scale[0].region, st->scale[0].base, 285 st->scale[0].base + st->scale[0].length - 1); 286 if (st->scale[0].region < 0 || st->scale[0].base == U64_MAX || 287 st->scale[0].length == U32_MAX) 288 return -EINVAL; 289 info->region_size[st->scale[0].region] = 290 max(info->region_size[st->scale[0].region], 291 st->scale[0].base + st->scale[0].length); 292 } 293 294 len = feat_matrix_length(info, &st->ofm, st->ofm.width, 295 st->ofm.height[2], st->ofm.depth); 296 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n", 297 op, st->ofm.region, st->ofm.base[0], len); 298 if (len == U64_MAX) 299 return -EINVAL; 300 info->output_region[st->ofm.region] = true; 301 302 return 0; 303 } 304 305 static int calc_sizes_elemwise(struct drm_device *ddev, 306 struct ethosu_validated_cmdstream_info *info, 307 u16 op, struct cmd_state *st, 308 bool ifm, bool ifm2) 309 { 310 u32 height, width, depth; 311 u64 len; 312 313 if (ifm) { 314 height = st->ifm.broadcast & 0x1 ? 0 : st->ofm.height[2]; 315 width = st->ifm.broadcast & 0x2 ? 0 : st->ofm.width; 316 depth = st->ifm.broadcast & 0x4 ? 0 : st->ofm.depth; 317 318 len = feat_matrix_length(info, &st->ifm, width, 319 height, depth); 320 dev_dbg(ddev->dev, "op %d: IFM:%d:0x%llx-0x%llx\n", 321 op, st->ifm.region, st->ifm.base[0], len); 322 if (len == U64_MAX) 323 return -EINVAL; 324 } 325 326 if (ifm2) { 327 height = st->ifm2.broadcast & 0x1 ? 0 : st->ofm.height[2]; 328 width = st->ifm2.broadcast & 0x2 ? 0 : st->ofm.width; 329 depth = st->ifm2.broadcast & 0x4 ? 0 : st->ofm.depth; 330 331 len = feat_matrix_length(info, &st->ifm2, width, 332 height, depth); 333 dev_dbg(ddev->dev, "op %d: IFM2:%d:0x%llx-0x%llx\n", 334 op, st->ifm2.region, st->ifm2.base[0], len); 335 if (len == U64_MAX) 336 return -EINVAL; 337 } 338 339 len = feat_matrix_length(info, &st->ofm, st->ofm.width, 340 st->ofm.height[2], st->ofm.depth); 341 dev_dbg(ddev->dev, "op %d: OFM:%d:0x%llx-0x%llx\n", 342 op, st->ofm.region, st->ofm.base[0], len); 343 if (len == U64_MAX) 344 return -EINVAL; 345 info->output_region[st->ofm.region] = true; 346 347 return 0; 348 } 349 350 static int ethosu_gem_cmdstream_copy_and_validate(struct drm_device *ddev, 351 u32 __user *ucmds, 352 struct ethosu_gem_object *bo, 353 u32 size) 354 { 355 struct ethosu_validated_cmdstream_info __free(kfree) *info = kzalloc_obj(*info, 356 GFP_KERNEL); 357 struct ethosu_device *edev = to_ethosu_device(ddev); 358 u32 *bocmds = bo->base.vaddr; 359 struct cmd_state st; 360 int i, ret; 361 362 if (!info) 363 return -ENOMEM; 364 info->cmd_size = size; 365 366 cmd_state_init(&st); 367 368 for (i = 0; i < size / 4; i++) { 369 bool use_ifm, use_ifm2, use_scale; 370 u64 dstlen, srclen; 371 u16 cmd, param; 372 u32 cmds[2]; 373 u64 addr; 374 375 if (get_user(cmds[0], ucmds++)) 376 return -EFAULT; 377 378 bocmds[i] = cmds[0]; 379 380 cmd = cmds[0]; 381 param = cmds[0] >> 16; 382 383 if (cmd & 0x4000) { 384 if (get_user(cmds[1], ucmds++)) 385 return -EFAULT; 386 387 i++; 388 bocmds[i] = cmds[1]; 389 addr = cmd_to_addr(cmds); 390 } 391 392 switch (cmd) { 393 case NPU_OP_DMA_START: 394 srclen = dma_length(info, &st.dma, &st.dma.src); 395 dstlen = dma_length(info, &st.dma, &st.dma.dst); 396 397 if (st.dma.dst.region >= 0) 398 info->output_region[st.dma.dst.region] = true; 399 dev_dbg(ddev->dev, "cmd: DMA SRC:%d:0x%llx+0x%llx DST:%d:0x%llx+0x%llx\n", 400 st.dma.src.region, st.dma.src.offset, srclen, 401 st.dma.dst.region, st.dma.dst.offset, dstlen); 402 break; 403 case NPU_OP_CONV: 404 case NPU_OP_DEPTHWISE: 405 use_ifm2 = param & 0x1; // weights_ifm2 406 use_scale = !(st.ofm.precision & 0x100); 407 ret = calc_sizes(ddev, info, cmd, &st, true, use_ifm2, 408 !use_ifm2, use_scale); 409 if (ret) 410 return ret; 411 break; 412 case NPU_OP_POOL: 413 use_ifm = param != 0x4; // pooling mode 414 use_scale = !(st.ofm.precision & 0x100); 415 ret = calc_sizes(ddev, info, cmd, &st, use_ifm, false, 416 false, use_scale); 417 if (ret) 418 return ret; 419 break; 420 case NPU_OP_ELEMENTWISE: 421 use_ifm2 = !((st.ifm2.broadcast == 8) || (param == 5) || 422 (param == 6) || (param == 7) || (param == 0x24)); 423 use_ifm = st.ifm.broadcast != 8; 424 ret = calc_sizes_elemwise(ddev, info, cmd, &st, use_ifm, use_ifm2); 425 if (ret) 426 return ret; 427 break; 428 case NPU_OP_RESIZE: // U85 only 429 WARN_ON(1); // TODO 430 break; 431 case NPU_SET_KERNEL_WIDTH_M1: 432 st.ifm.width = param; 433 break; 434 case NPU_SET_KERNEL_HEIGHT_M1: 435 st.ifm.height[2] = param; 436 break; 437 case NPU_SET_KERNEL_STRIDE: 438 st.ifm.stride_kernel = param; 439 break; 440 case NPU_SET_IFM_PAD_TOP: 441 st.ifm.pad_top = param & 0x7f; 442 break; 443 case NPU_SET_IFM_PAD_LEFT: 444 st.ifm.pad_left = param & 0x7f; 445 break; 446 case NPU_SET_IFM_PAD_RIGHT: 447 st.ifm.pad_right = param & 0xff; 448 break; 449 case NPU_SET_IFM_PAD_BOTTOM: 450 st.ifm.pad_bottom = param & 0xff; 451 break; 452 case NPU_SET_IFM_DEPTH_M1: 453 st.ifm.depth = param; 454 break; 455 case NPU_SET_IFM_PRECISION: 456 st.ifm.precision = param; 457 break; 458 case NPU_SET_IFM_BROADCAST: 459 st.ifm.broadcast = param; 460 break; 461 case NPU_SET_IFM_REGION: 462 st.ifm.region = param & 0x7f; 463 break; 464 case NPU_SET_IFM_WIDTH0_M1: 465 st.ifm.width0 = param; 466 break; 467 case NPU_SET_IFM_HEIGHT0_M1: 468 st.ifm.height[0] = param; 469 break; 470 case NPU_SET_IFM_HEIGHT1_M1: 471 st.ifm.height[1] = param; 472 break; 473 case NPU_SET_IFM_BASE0: 474 case NPU_SET_IFM_BASE1: 475 case NPU_SET_IFM_BASE2: 476 case NPU_SET_IFM_BASE3: 477 st.ifm.base[cmd & 0x3] = addr; 478 break; 479 case NPU_SET_IFM_STRIDE_X: 480 st.ifm.stride_x = addr; 481 break; 482 case NPU_SET_IFM_STRIDE_Y: 483 st.ifm.stride_y = addr; 484 break; 485 case NPU_SET_IFM_STRIDE_C: 486 st.ifm.stride_c = addr; 487 break; 488 489 case NPU_SET_OFM_WIDTH_M1: 490 st.ofm.width = param; 491 break; 492 case NPU_SET_OFM_HEIGHT_M1: 493 st.ofm.height[2] = param; 494 break; 495 case NPU_SET_OFM_DEPTH_M1: 496 st.ofm.depth = param; 497 break; 498 case NPU_SET_OFM_PRECISION: 499 st.ofm.precision = param; 500 break; 501 case NPU_SET_OFM_REGION: 502 st.ofm.region = param & 0x7; 503 break; 504 case NPU_SET_OFM_WIDTH0_M1: 505 st.ofm.width0 = param; 506 break; 507 case NPU_SET_OFM_HEIGHT0_M1: 508 st.ofm.height[0] = param; 509 break; 510 case NPU_SET_OFM_HEIGHT1_M1: 511 st.ofm.height[1] = param; 512 break; 513 case NPU_SET_OFM_BASE0: 514 case NPU_SET_OFM_BASE1: 515 case NPU_SET_OFM_BASE2: 516 case NPU_SET_OFM_BASE3: 517 st.ofm.base[cmd & 0x3] = addr; 518 break; 519 case NPU_SET_OFM_STRIDE_X: 520 st.ofm.stride_x = addr; 521 break; 522 case NPU_SET_OFM_STRIDE_Y: 523 st.ofm.stride_y = addr; 524 break; 525 case NPU_SET_OFM_STRIDE_C: 526 st.ofm.stride_c = addr; 527 break; 528 529 case NPU_SET_IFM2_BROADCAST: 530 st.ifm2.broadcast = param; 531 break; 532 case NPU_SET_IFM2_PRECISION: 533 st.ifm2.precision = param; 534 break; 535 case NPU_SET_IFM2_REGION: 536 st.ifm2.region = param & 0x7; 537 break; 538 case NPU_SET_IFM2_WIDTH0_M1: 539 st.ifm2.width0 = param; 540 break; 541 case NPU_SET_IFM2_HEIGHT0_M1: 542 st.ifm2.height[0] = param; 543 break; 544 case NPU_SET_IFM2_HEIGHT1_M1: 545 st.ifm2.height[1] = param; 546 break; 547 case NPU_SET_IFM2_BASE0: 548 case NPU_SET_IFM2_BASE1: 549 case NPU_SET_IFM2_BASE2: 550 case NPU_SET_IFM2_BASE3: 551 st.ifm2.base[cmd & 0x3] = addr; 552 break; 553 case NPU_SET_IFM2_STRIDE_X: 554 st.ifm2.stride_x = addr; 555 break; 556 case NPU_SET_IFM2_STRIDE_Y: 557 st.ifm2.stride_y = addr; 558 break; 559 case NPU_SET_IFM2_STRIDE_C: 560 st.ifm2.stride_c = addr; 561 break; 562 563 case NPU_SET_WEIGHT_REGION: 564 st.weight[0].region = param & 0x7; 565 break; 566 case NPU_SET_SCALE_REGION: 567 st.scale[0].region = param & 0x7; 568 break; 569 case NPU_SET_WEIGHT_BASE: 570 st.weight[0].base = addr; 571 break; 572 case NPU_SET_WEIGHT_LENGTH: 573 st.weight[0].length = cmds[1]; 574 break; 575 case NPU_SET_SCALE_BASE: 576 st.scale[0].base = addr; 577 break; 578 case NPU_SET_SCALE_LENGTH: 579 st.scale[0].length = cmds[1]; 580 break; 581 case NPU_SET_WEIGHT1_BASE: 582 st.weight[1].base = addr; 583 break; 584 case NPU_SET_WEIGHT1_LENGTH: 585 st.weight[1].length = cmds[1]; 586 break; 587 case NPU_SET_SCALE1_BASE: // NPU_SET_WEIGHT2_BASE (U85) 588 if (ethosu_is_u65(edev)) 589 st.scale[1].base = addr; 590 else 591 st.weight[2].base = addr; 592 break; 593 case NPU_SET_SCALE1_LENGTH: // NPU_SET_WEIGHT2_LENGTH (U85) 594 if (ethosu_is_u65(edev)) 595 st.scale[1].length = cmds[1]; 596 else 597 st.weight[1].length = cmds[1]; 598 break; 599 case NPU_SET_WEIGHT3_BASE: 600 st.weight[3].base = addr; 601 break; 602 case NPU_SET_WEIGHT3_LENGTH: 603 st.weight[3].length = cmds[1]; 604 break; 605 606 case NPU_SET_DMA0_SRC_REGION: 607 if (param & 0x100) 608 st.dma.src.region = -1; 609 else 610 st.dma.src.region = param & 0x7; 611 st.dma.mode = (param >> 9) & 0x3; 612 break; 613 case NPU_SET_DMA0_DST_REGION: 614 if (param & 0x100) 615 st.dma.dst.region = -1; 616 else 617 st.dma.dst.region = param & 0x7; 618 break; 619 case NPU_SET_DMA0_SIZE0: 620 st.dma.size0 = param; 621 break; 622 case NPU_SET_DMA0_SIZE1: 623 st.dma.size1 = param; 624 break; 625 case NPU_SET_DMA0_SRC_STRIDE0: 626 st.dma.src.stride[0] = ((s64)addr << 24) >> 24; 627 break; 628 case NPU_SET_DMA0_SRC_STRIDE1: 629 st.dma.src.stride[1] = ((s64)addr << 24) >> 24; 630 break; 631 case NPU_SET_DMA0_DST_STRIDE0: 632 st.dma.dst.stride[0] = ((s64)addr << 24) >> 24; 633 break; 634 case NPU_SET_DMA0_DST_STRIDE1: 635 st.dma.dst.stride[1] = ((s64)addr << 24) >> 24; 636 break; 637 case NPU_SET_DMA0_SRC: 638 st.dma.src.offset = addr; 639 break; 640 case NPU_SET_DMA0_DST: 641 st.dma.dst.offset = addr; 642 break; 643 case NPU_SET_DMA0_LEN: 644 st.dma.src.len = st.dma.dst.len = addr; 645 break; 646 default: 647 break; 648 } 649 } 650 651 for (i = 0; i < NPU_BASEP_REGION_MAX; i++) { 652 if (!info->region_size[i]) 653 continue; 654 dev_dbg(ddev->dev, "region %d max size: 0x%llx\n", 655 i, info->region_size[i]); 656 } 657 658 bo->info = no_free_ptr(info); 659 return 0; 660 } 661 662 /** 663 * ethosu_gem_cmdstream_create() - Create a GEM object and attach it to a handle. 664 * @file: DRM file. 665 * @ddev: DRM device. 666 * @exclusive_vm: Exclusive VM. Not NULL if the GEM object can't be shared. 667 * @size: Size of the GEM object to allocate. 668 * @flags: Combination of drm_ethosu_bo_flags flags. 669 * @handle: Pointer holding the handle pointing to the new GEM object. 670 * 671 * Return: Zero on success 672 */ 673 int ethosu_gem_cmdstream_create(struct drm_file *file, 674 struct drm_device *ddev, 675 u32 size, u64 data, u32 flags, u32 *handle) 676 { 677 int ret; 678 struct drm_gem_dma_object *mem; 679 struct ethosu_gem_object *bo; 680 681 mem = drm_gem_dma_create(ddev, size); 682 if (IS_ERR(mem)) 683 return PTR_ERR(mem); 684 685 bo = to_ethosu_bo(&mem->base); 686 bo->flags = flags; 687 688 ret = ethosu_gem_cmdstream_copy_and_validate(ddev, 689 (void __user *)(uintptr_t)data, 690 bo, size); 691 if (ret) 692 goto fail; 693 694 /* 695 * Allocate an id of idr table where the obj is registered 696 * and handle has the id what user can see. 697 */ 698 ret = drm_gem_handle_create(file, &mem->base, handle); 699 700 fail: 701 /* drop reference from allocate - handle holds it now. */ 702 drm_gem_object_put(&mem->base); 703 704 return ret; 705 } 706