1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * A virtual stateless decoder device for stateless uAPI development purposes. 4 * 5 * This tool's objective is to help the development and testing of userspace 6 * applications that use the V4L2 stateless API to decode media. 7 * 8 * A userspace implementation can use visl to run a decoding loop even when no 9 * hardware is available or when the kernel uAPI for the codec has not been 10 * upstreamed yet. This can reveal bugs at an early stage. 11 * 12 * This driver can also trace the contents of the V4L2 controls submitted to it. 13 * It can also dump the contents of the vb2 buffers through a debugfs 14 * interface. This is in many ways similar to the tracing infrastructure 15 * available for other popular encode/decode APIs out there and can help develop 16 * a userspace application by using another (working) one as a reference. 17 * 18 * Note that no actual decoding of video frames is performed by visl. The V4L2 19 * test pattern generator is used to write various debug information to the 20 * capture buffers instead. 21 * 22 * Copyright (C) 2022 Collabora, Ltd. 23 * 24 * Based on the vim2m driver, that is: 25 * 26 * Copyright (c) 2009-2010 Samsung Electronics Co., Ltd. 27 * Pawel Osciak, <pawel@osciak.com> 28 * Marek Szyprowski, <m.szyprowski@samsung.com> 29 * 30 * Based on the vicodec driver, that is: 31 * 32 * Copyright 2018 Cisco Systems, Inc. and/or its affiliates. All rights reserved. 33 * 34 * Based on the Cedrus VPU driver, that is: 35 * 36 * Copyright (C) 2016 Florent Revest <florent.revest@free-electrons.com> 37 * Copyright (C) 2018 Paul Kocialkowski <paul.kocialkowski@bootlin.com> 38 * Copyright (C) 2018 Bootlin 39 */ 40 41 #include <linux/debugfs.h> 42 #include <linux/module.h> 43 #include <linux/platform_device.h> 44 #include <media/v4l2-ctrls.h> 45 #include <media/v4l2-device.h> 46 #include <media/v4l2-ioctl.h> 47 #include <media/v4l2-mem2mem.h> 48 49 #include "visl.h" 50 #include "visl-dec.h" 51 #include "visl-debugfs.h" 52 #include "visl-video.h" 53 54 unsigned int visl_debug; 55 module_param(visl_debug, uint, 0644); 56 MODULE_PARM_DESC(visl_debug, " activates debug info"); 57 58 unsigned int visl_transtime_ms; 59 module_param(visl_transtime_ms, uint, 0644); 60 MODULE_PARM_DESC(visl_transtime_ms, " simulated process time in milliseconds."); 61 62 /* 63 * dprintk can be slow through serial. This lets one limit the tracing to a 64 * particular number of frames 65 */ 66 int visl_dprintk_frame_start = -1; 67 module_param(visl_dprintk_frame_start, int, 0444); 68 MODULE_PARM_DESC(visl_dprintk_frame_start, 69 " a frame number to start tracing with dprintk"); 70 71 unsigned int visl_dprintk_nframes; 72 module_param(visl_dprintk_nframes, uint, 0444); 73 MODULE_PARM_DESC(visl_dprintk_nframes, 74 " the number of frames to trace with dprintk"); 75 76 bool keep_bitstream_buffers; 77 module_param(keep_bitstream_buffers, bool, 0444); 78 MODULE_PARM_DESC(keep_bitstream_buffers, 79 " keep bitstream buffers in debugfs after streaming is stopped"); 80 81 int bitstream_trace_frame_start = -1; 82 module_param(bitstream_trace_frame_start, int, 0444); 83 MODULE_PARM_DESC(bitstream_trace_frame_start, 84 " a frame number to start dumping the bitstream through debugfs"); 85 86 unsigned int bitstream_trace_nframes; 87 module_param(bitstream_trace_nframes, uint, 0444); 88 MODULE_PARM_DESC(bitstream_trace_nframes, 89 " the number of frames to dump the bitstream through debugfs"); 90 91 bool tpg_verbose; 92 module_param(tpg_verbose, bool, 0644); 93 MODULE_PARM_DESC(tpg_verbose, 94 " add more verbose information on the generated output frames"); 95 96 static const struct visl_ctrl_desc visl_fwht_ctrl_descs[] = { 97 { 98 .cfg.id = V4L2_CID_STATELESS_FWHT_PARAMS, 99 }, 100 }; 101 102 const struct visl_ctrls visl_fwht_ctrls = { 103 .ctrls = visl_fwht_ctrl_descs, 104 .num_ctrls = ARRAY_SIZE(visl_fwht_ctrl_descs) 105 }; 106 107 static const struct visl_ctrl_desc visl_mpeg2_ctrl_descs[] = { 108 { 109 .cfg.id = V4L2_CID_STATELESS_MPEG2_SEQUENCE, 110 }, 111 { 112 .cfg.id = V4L2_CID_STATELESS_MPEG2_PICTURE, 113 }, 114 { 115 .cfg.id = V4L2_CID_STATELESS_MPEG2_QUANTISATION, 116 }, 117 }; 118 119 const struct visl_ctrls visl_mpeg2_ctrls = { 120 .ctrls = visl_mpeg2_ctrl_descs, 121 .num_ctrls = ARRAY_SIZE(visl_mpeg2_ctrl_descs), 122 }; 123 124 static const struct visl_ctrl_desc visl_vp8_ctrl_descs[] = { 125 { 126 .cfg.id = V4L2_CID_STATELESS_VP8_FRAME, 127 }, 128 }; 129 130 const struct visl_ctrls visl_vp8_ctrls = { 131 .ctrls = visl_vp8_ctrl_descs, 132 .num_ctrls = ARRAY_SIZE(visl_vp8_ctrl_descs), 133 }; 134 135 static const struct visl_ctrl_desc visl_vp9_ctrl_descs[] = { 136 { 137 .cfg.id = V4L2_CID_STATELESS_VP9_FRAME, 138 }, 139 { 140 .cfg.id = V4L2_CID_STATELESS_VP9_COMPRESSED_HDR, 141 }, 142 }; 143 144 const struct visl_ctrls visl_vp9_ctrls = { 145 .ctrls = visl_vp9_ctrl_descs, 146 .num_ctrls = ARRAY_SIZE(visl_vp9_ctrl_descs), 147 }; 148 149 static const struct visl_ctrl_desc visl_h264_ctrl_descs[] = { 150 { 151 .cfg.id = V4L2_CID_STATELESS_H264_DECODE_PARAMS, 152 }, 153 { 154 .cfg.id = V4L2_CID_STATELESS_H264_SPS, 155 }, 156 { 157 .cfg.id = V4L2_CID_STATELESS_H264_PPS, 158 }, 159 { 160 .cfg.id = V4L2_CID_STATELESS_H264_SCALING_MATRIX, 161 }, 162 { 163 .cfg.id = V4L2_CID_STATELESS_H264_DECODE_MODE, 164 .cfg.min = V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED, 165 .cfg.max = V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED, 166 .cfg.def = V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED, 167 }, 168 { 169 .cfg.id = V4L2_CID_STATELESS_H264_START_CODE, 170 .cfg.min = V4L2_STATELESS_H264_START_CODE_NONE, 171 .cfg.max = V4L2_STATELESS_H264_START_CODE_ANNEX_B, 172 .cfg.def = V4L2_STATELESS_H264_START_CODE_NONE, 173 }, 174 { 175 .cfg.id = V4L2_CID_STATELESS_H264_SLICE_PARAMS, 176 }, 177 { 178 .cfg.id = V4L2_CID_STATELESS_H264_PRED_WEIGHTS, 179 }, 180 }; 181 182 const struct visl_ctrls visl_h264_ctrls = { 183 .ctrls = visl_h264_ctrl_descs, 184 .num_ctrls = ARRAY_SIZE(visl_h264_ctrl_descs), 185 }; 186 187 static const struct visl_ctrl_desc visl_hevc_ctrl_descs[] = { 188 { 189 .cfg.id = V4L2_CID_STATELESS_HEVC_SPS, 190 }, 191 { 192 .cfg.id = V4L2_CID_STATELESS_HEVC_PPS, 193 }, 194 { 195 .cfg.id = V4L2_CID_STATELESS_HEVC_SLICE_PARAMS, 196 /* The absolute maximum for level > 6 */ 197 .cfg.dims = { 600 }, 198 }, 199 { 200 .cfg.id = V4L2_CID_STATELESS_HEVC_SCALING_MATRIX, 201 }, 202 { 203 .cfg.id = V4L2_CID_STATELESS_HEVC_DECODE_PARAMS, 204 }, 205 { 206 .cfg.id = V4L2_CID_STATELESS_HEVC_DECODE_MODE, 207 .cfg.min = V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED, 208 .cfg.max = V4L2_STATELESS_HEVC_DECODE_MODE_FRAME_BASED, 209 .cfg.def = V4L2_STATELESS_HEVC_DECODE_MODE_SLICE_BASED, 210 }, 211 { 212 .cfg.id = V4L2_CID_STATELESS_HEVC_START_CODE, 213 .cfg.min = V4L2_STATELESS_HEVC_START_CODE_NONE, 214 .cfg.max = V4L2_STATELESS_HEVC_START_CODE_ANNEX_B, 215 .cfg.def = V4L2_STATELESS_HEVC_START_CODE_NONE, 216 }, 217 { 218 .cfg.id = V4L2_CID_STATELESS_HEVC_ENTRY_POINT_OFFSETS, 219 .cfg.dims = { 256 }, 220 .cfg.max = 0xffffffff, 221 .cfg.step = 1, 222 }, 223 224 }; 225 226 const struct visl_ctrls visl_hevc_ctrls = { 227 .ctrls = visl_hevc_ctrl_descs, 228 .num_ctrls = ARRAY_SIZE(visl_hevc_ctrl_descs), 229 }; 230 231 static const struct visl_ctrl_desc visl_av1_ctrl_descs[] = { 232 { 233 .cfg.id = V4L2_CID_STATELESS_AV1_FRAME, 234 }, 235 { 236 .cfg.id = V4L2_CID_STATELESS_AV1_TILE_GROUP_ENTRY, 237 .cfg.dims = { V4L2_AV1_MAX_TILE_COUNT }, 238 }, 239 { 240 .cfg.id = V4L2_CID_STATELESS_AV1_SEQUENCE, 241 }, 242 { 243 .cfg.id = V4L2_CID_STATELESS_AV1_FILM_GRAIN, 244 }, 245 }; 246 247 const struct visl_ctrls visl_av1_ctrls = { 248 .ctrls = visl_av1_ctrl_descs, 249 .num_ctrls = ARRAY_SIZE(visl_av1_ctrl_descs), 250 }; 251 252 struct v4l2_ctrl *visl_find_control(struct visl_ctx *ctx, u32 id) 253 { 254 struct v4l2_ctrl_handler *hdl = &ctx->hdl; 255 256 return v4l2_ctrl_find(hdl, id); 257 } 258 259 void *visl_find_control_data(struct visl_ctx *ctx, u32 id) 260 { 261 struct v4l2_ctrl *ctrl; 262 263 ctrl = visl_find_control(ctx, id); 264 if (ctrl) 265 return ctrl->p_cur.p; 266 267 return NULL; 268 } 269 270 u32 visl_control_num_elems(struct visl_ctx *ctx, u32 id) 271 { 272 struct v4l2_ctrl *ctrl; 273 274 ctrl = visl_find_control(ctx, id); 275 if (ctrl) 276 return ctrl->elems; 277 278 return 0; 279 } 280 281 static void visl_device_release(struct video_device *vdev) 282 { 283 struct visl_dev *dev = container_of(vdev, struct visl_dev, vfd); 284 285 v4l2_device_unregister(&dev->v4l2_dev); 286 v4l2_m2m_release(dev->m2m_dev); 287 media_device_cleanup(&dev->mdev); 288 visl_debugfs_deinit(dev); 289 kfree(dev); 290 } 291 292 #define VISL_CONTROLS_COUNT ARRAY_SIZE(visl_controls) 293 294 static int visl_init_ctrls(struct visl_ctx *ctx) 295 { 296 struct visl_dev *dev = ctx->dev; 297 struct v4l2_ctrl_handler *hdl = &ctx->hdl; 298 unsigned int ctrl_cnt = 0; 299 unsigned int i; 300 unsigned int j; 301 const struct visl_ctrls *ctrls; 302 303 for (i = 0; i < num_coded_fmts; i++) 304 ctrl_cnt += visl_coded_fmts[i].ctrls->num_ctrls; 305 306 v4l2_ctrl_handler_init(hdl, ctrl_cnt); 307 308 for (i = 0; i < num_coded_fmts; i++) { 309 ctrls = visl_coded_fmts[i].ctrls; 310 for (j = 0; j < ctrls->num_ctrls; j++) 311 v4l2_ctrl_new_custom(hdl, &ctrls->ctrls[j].cfg, NULL); 312 } 313 314 if (hdl->error) { 315 v4l2_err(&dev->v4l2_dev, 316 "Failed to initialize control handler\n"); 317 v4l2_ctrl_handler_free(hdl); 318 return hdl->error; 319 } 320 321 ctx->fh.ctrl_handler = hdl; 322 v4l2_ctrl_handler_setup(hdl); 323 324 return 0; 325 } 326 327 static int visl_open(struct file *file) 328 { 329 struct visl_dev *dev = video_drvdata(file); 330 struct visl_ctx *ctx = NULL; 331 int rc = 0; 332 333 if (mutex_lock_interruptible(&dev->dev_mutex)) 334 return -ERESTARTSYS; 335 ctx = kzalloc_obj(*ctx); 336 if (!ctx) { 337 rc = -ENOMEM; 338 goto unlock; 339 } 340 341 ctx->tpg_str_buf = kzalloc(TPG_STR_BUF_SZ, GFP_KERNEL); 342 if (!ctx->tpg_str_buf) { 343 rc = -ENOMEM; 344 goto free_ctx; 345 } 346 347 v4l2_fh_init(&ctx->fh, video_devdata(file)); 348 ctx->dev = dev; 349 350 rc = visl_init_ctrls(ctx); 351 if (rc) 352 goto free_ctx; 353 354 ctx->fh.m2m_ctx = v4l2_m2m_ctx_init(dev->m2m_dev, ctx, &visl_queue_init); 355 356 mutex_init(&ctx->vb_mutex); 357 358 if (IS_ERR(ctx->fh.m2m_ctx)) { 359 rc = PTR_ERR(ctx->fh.m2m_ctx); 360 goto free_hdl; 361 } 362 363 rc = visl_set_default_format(ctx); 364 if (rc) 365 goto free_m2m_ctx; 366 367 v4l2_fh_add(&ctx->fh, file); 368 369 dprintk(dev, "Created instance: %p, m2m_ctx: %p\n", 370 ctx, ctx->fh.m2m_ctx); 371 372 mutex_unlock(&dev->dev_mutex); 373 return rc; 374 375 free_m2m_ctx: 376 v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); 377 free_hdl: 378 v4l2_ctrl_handler_free(&ctx->hdl); 379 v4l2_fh_exit(&ctx->fh); 380 free_ctx: 381 kfree(ctx->tpg_str_buf); 382 kfree(ctx); 383 unlock: 384 mutex_unlock(&dev->dev_mutex); 385 return rc; 386 } 387 388 static int visl_release(struct file *file) 389 { 390 struct visl_dev *dev = video_drvdata(file); 391 struct visl_ctx *ctx = visl_file_to_ctx(file); 392 393 dprintk(dev, "Releasing instance %p\n", ctx); 394 395 tpg_free(&ctx->tpg); 396 v4l2_fh_del(&ctx->fh, file); 397 v4l2_fh_exit(&ctx->fh); 398 v4l2_ctrl_handler_free(&ctx->hdl); 399 mutex_lock(&dev->dev_mutex); 400 v4l2_m2m_ctx_release(ctx->fh.m2m_ctx); 401 mutex_unlock(&dev->dev_mutex); 402 403 kfree(ctx->tpg_str_buf); 404 kfree(ctx); 405 406 return 0; 407 } 408 409 static const struct v4l2_file_operations visl_fops = { 410 .owner = THIS_MODULE, 411 .open = visl_open, 412 .release = visl_release, 413 .poll = v4l2_m2m_fop_poll, 414 .unlocked_ioctl = video_ioctl2, 415 .mmap = v4l2_m2m_fop_mmap, 416 }; 417 418 static const struct video_device visl_videodev = { 419 .name = VISL_NAME, 420 .vfl_dir = VFL_DIR_M2M, 421 .fops = &visl_fops, 422 .ioctl_ops = &visl_ioctl_ops, 423 .minor = -1, 424 .release = visl_device_release, 425 .device_caps = V4L2_CAP_VIDEO_M2M_MPLANE | V4L2_CAP_STREAMING, 426 }; 427 428 static const struct v4l2_m2m_ops visl_m2m_ops = { 429 .device_run = visl_device_run, 430 }; 431 432 static const struct media_device_ops visl_m2m_media_ops = { 433 .req_validate = visl_request_validate, 434 .req_queue = v4l2_m2m_request_queue, 435 }; 436 437 static int visl_probe(struct platform_device *pdev) 438 { 439 struct visl_dev *dev; 440 struct video_device *vfd; 441 int ret; 442 int rc; 443 444 dev = kzalloc_obj(*dev); 445 if (!dev) 446 return -ENOMEM; 447 448 ret = v4l2_device_register(&pdev->dev, &dev->v4l2_dev); 449 if (ret) 450 goto error_visl_dev; 451 452 mutex_init(&dev->dev_mutex); 453 454 dev->vfd = visl_videodev; 455 vfd = &dev->vfd; 456 vfd->lock = &dev->dev_mutex; 457 vfd->v4l2_dev = &dev->v4l2_dev; 458 459 video_set_drvdata(vfd, dev); 460 461 platform_set_drvdata(pdev, dev); 462 463 dev->m2m_dev = v4l2_m2m_init(&visl_m2m_ops); 464 if (IS_ERR(dev->m2m_dev)) { 465 v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem device\n"); 466 ret = PTR_ERR(dev->m2m_dev); 467 dev->m2m_dev = NULL; 468 goto error_dev; 469 } 470 471 dev->mdev.dev = &pdev->dev; 472 strscpy(dev->mdev.model, "visl", sizeof(dev->mdev.model)); 473 strscpy(dev->mdev.bus_info, "platform:visl", 474 sizeof(dev->mdev.bus_info)); 475 media_device_init(&dev->mdev); 476 dev->mdev.ops = &visl_m2m_media_ops; 477 dev->v4l2_dev.mdev = &dev->mdev; 478 479 ret = video_register_device(vfd, VFL_TYPE_VIDEO, -1); 480 if (ret) { 481 v4l2_err(&dev->v4l2_dev, "Failed to register video device\n"); 482 goto error_m2m; 483 } 484 485 v4l2_info(&dev->v4l2_dev, 486 "Device registered as /dev/video%d\n", vfd->num); 487 488 ret = v4l2_m2m_register_media_controller(dev->m2m_dev, vfd, 489 MEDIA_ENT_F_PROC_VIDEO_DECODER); 490 if (ret) { 491 v4l2_err(&dev->v4l2_dev, "Failed to init mem2mem media controller\n"); 492 goto error_v4l2; 493 } 494 495 ret = media_device_register(&dev->mdev); 496 if (ret) { 497 v4l2_err(&dev->v4l2_dev, "Failed to register mem2mem media device\n"); 498 goto error_m2m_mc; 499 } 500 501 rc = visl_debugfs_init(dev); 502 if (rc) 503 dprintk(dev, "visl_debugfs_init failed: %d\n" 504 "Continuing without debugfs support\n", rc); 505 506 return 0; 507 508 error_m2m_mc: 509 v4l2_m2m_unregister_media_controller(dev->m2m_dev); 510 error_v4l2: 511 video_unregister_device(&dev->vfd); 512 /* visl_device_release called by video_unregister_device to release various objects */ 513 return ret; 514 error_m2m: 515 v4l2_m2m_release(dev->m2m_dev); 516 error_dev: 517 v4l2_device_unregister(&dev->v4l2_dev); 518 error_visl_dev: 519 kfree(dev); 520 521 return ret; 522 } 523 524 static void visl_remove(struct platform_device *pdev) 525 { 526 struct visl_dev *dev = platform_get_drvdata(pdev); 527 528 v4l2_info(&dev->v4l2_dev, "Removing " VISL_NAME); 529 530 #ifdef CONFIG_MEDIA_CONTROLLER 531 if (media_devnode_is_registered(dev->mdev.devnode)) { 532 media_device_unregister(&dev->mdev); 533 v4l2_m2m_unregister_media_controller(dev->m2m_dev); 534 } 535 #endif 536 video_unregister_device(&dev->vfd); 537 } 538 539 static struct platform_driver visl_pdrv = { 540 .probe = visl_probe, 541 .remove = visl_remove, 542 .driver = { 543 .name = VISL_NAME, 544 }, 545 }; 546 547 static void visl_dev_release(struct device *dev) {} 548 549 static struct platform_device visl_pdev = { 550 .name = VISL_NAME, 551 .dev.release = visl_dev_release, 552 }; 553 554 static void __exit visl_exit(void) 555 { 556 platform_driver_unregister(&visl_pdrv); 557 platform_device_unregister(&visl_pdev); 558 } 559 560 static int __init visl_init(void) 561 { 562 int ret; 563 564 ret = platform_device_register(&visl_pdev); 565 if (ret) 566 return ret; 567 568 ret = platform_driver_register(&visl_pdrv); 569 if (ret) 570 platform_device_unregister(&visl_pdev); 571 572 return ret; 573 } 574 575 MODULE_DESCRIPTION("Virtual stateless decoder device"); 576 MODULE_AUTHOR("Daniel Almeida <daniel.almeida@collabora.com>"); 577 MODULE_LICENSE("GPL"); 578 579 module_init(visl_init); 580 module_exit(visl_exit); 581