1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Hantro VPU codec driver
4 *
5 * Copyright 2018 Google LLC.
6 * Tomasz Figa <tfiga@chromium.org>
7 */
8
9 #ifndef HANTRO_HW_H_
10 #define HANTRO_HW_H_
11
12 #include <linux/interrupt.h>
13 #include <linux/v4l2-controls.h>
14 #include <media/v4l2-ctrls.h>
15 #include <media/v4l2-vp9.h>
16 #include <media/videobuf2-core.h>
17
18 #include "rockchip_av1_entropymode.h"
19 #include "rockchip_av1_filmgrain.h"
20
21 #define DEC_8190_ALIGN_MASK 0x07U
22
23 #define MB_DIM 16
24 #define TILE_MB_DIM 4
25 #define MB_WIDTH(w) DIV_ROUND_UP(w, MB_DIM)
26 #define MB_HEIGHT(h) DIV_ROUND_UP(h, MB_DIM)
27
28 #define FMT_MIN_WIDTH 48
29 #define FMT_MIN_HEIGHT 48
30 #define FMT_HD_WIDTH 1280
31 #define FMT_HD_HEIGHT 720
32 #define FMT_FHD_WIDTH 1920
33 #define FMT_FHD_HEIGHT 1088
34 #define FMT_UHD_WIDTH 3840
35 #define FMT_UHD_HEIGHT 2160
36 #define FMT_4K_WIDTH 4096
37 #define FMT_4K_HEIGHT 2304
38
39 #define NUM_REF_PICTURES (V4L2_HEVC_DPB_ENTRIES_NUM_MAX + 1)
40
41 #define AV1_MAX_FRAME_BUF_COUNT (V4L2_AV1_TOTAL_REFS_PER_FRAME + 1)
42
43 #define MAX_POSTPROC_BUFFERS 64
44
45 struct hantro_dev;
46 struct hantro_ctx;
47 struct hantro_buf;
48 struct hantro_variant;
49
50 /**
51 * struct hantro_aux_buf - auxiliary DMA buffer for hardware data
52 *
53 * @cpu: CPU pointer to the buffer.
54 * @dma: DMA address of the buffer.
55 * @size: Size of the buffer.
56 * @attrs: Attributes of the DMA mapping.
57 */
58 struct hantro_aux_buf {
59 void *cpu;
60 dma_addr_t dma;
61 size_t size;
62 unsigned long attrs;
63 };
64
65 /* Max. number of entries in the DPB (HW limitation). */
66 #define HANTRO_H264_DPB_SIZE 16
67
68 /**
69 * struct hantro_h264_dec_ctrls
70 *
71 * @decode: Decode params
72 * @scaling: Scaling info
73 * @sps: SPS info
74 * @pps: PPS info
75 */
76 struct hantro_h264_dec_ctrls {
77 const struct v4l2_ctrl_h264_decode_params *decode;
78 const struct v4l2_ctrl_h264_scaling_matrix *scaling;
79 const struct v4l2_ctrl_h264_sps *sps;
80 const struct v4l2_ctrl_h264_pps *pps;
81 };
82
83 /**
84 * struct hantro_h264_dec_reflists
85 *
86 * @p: P reflist
87 * @b0: B0 reflist
88 * @b1: B1 reflist
89 */
90 struct hantro_h264_dec_reflists {
91 struct v4l2_h264_reference p[V4L2_H264_REF_LIST_LEN];
92 struct v4l2_h264_reference b0[V4L2_H264_REF_LIST_LEN];
93 struct v4l2_h264_reference b1[V4L2_H264_REF_LIST_LEN];
94 };
95
96 /**
97 * struct hantro_h264_dec_hw_ctx
98 *
99 * @priv: Private auxiliary buffer for hardware.
100 * @dpb: DPB
101 * @reflists: P/B0/B1 reflists
102 * @ctrls: V4L2 controls attached to a run
103 * @dpb_longterm: DPB long-term
104 * @dpb_valid: DPB valid
105 * @cur_poc: Current picture order count
106 */
107 struct hantro_h264_dec_hw_ctx {
108 struct hantro_aux_buf priv;
109 struct v4l2_h264_dpb_entry dpb[HANTRO_H264_DPB_SIZE];
110 struct hantro_h264_dec_reflists reflists;
111 struct hantro_h264_dec_ctrls ctrls;
112 u32 dpb_longterm;
113 u32 dpb_valid;
114 s32 cur_poc;
115 };
116
117 /**
118 * struct hantro_hevc_dec_ctrls
119 * @decode_params: Decode params
120 * @scaling: Scaling matrix
121 * @sps: SPS info
122 * @pps: PPS info
123 * @hevc_hdr_skip_length: the number of data (in bits) to skip in the
124 * slice segment header syntax after 'slice type'
125 * token
126 */
127 struct hantro_hevc_dec_ctrls {
128 const struct v4l2_ctrl_hevc_decode_params *decode_params;
129 const struct v4l2_ctrl_hevc_scaling_matrix *scaling;
130 const struct v4l2_ctrl_hevc_sps *sps;
131 const struct v4l2_ctrl_hevc_pps *pps;
132 u32 hevc_hdr_skip_length;
133 };
134
135 /**
136 * struct hantro_hevc_dec_hw_ctx
137 * @tile_sizes: Tile sizes buffer
138 * @tile_filter: Tile vertical filter buffer
139 * @tile_sao: Tile SAO buffer
140 * @tile_bsd: Tile BSD control buffer
141 * @ref_bufs: Internal reference buffers
142 * @scaling_lists: Scaling lists buffer
143 * @ref_bufs_poc: Internal reference buffers picture order count
144 * @ref_bufs_used: Bitfield of used reference buffers
145 * @ctrls: V4L2 controls attached to a run
146 * @num_tile_cols_allocated: number of allocated tiles
147 */
148 struct hantro_hevc_dec_hw_ctx {
149 struct hantro_aux_buf tile_sizes;
150 struct hantro_aux_buf tile_filter;
151 struct hantro_aux_buf tile_sao;
152 struct hantro_aux_buf tile_bsd;
153 struct hantro_aux_buf ref_bufs[NUM_REF_PICTURES];
154 struct hantro_aux_buf scaling_lists;
155 s32 ref_bufs_poc[NUM_REF_PICTURES];
156 u32 ref_bufs_used;
157 struct hantro_hevc_dec_ctrls ctrls;
158 unsigned int num_tile_cols_allocated;
159 };
160
161 /**
162 * struct hantro_mpeg2_dec_hw_ctx
163 *
164 * @qtable: Quantization table
165 */
166 struct hantro_mpeg2_dec_hw_ctx {
167 struct hantro_aux_buf qtable;
168 };
169
170 /**
171 * struct hantro_vp8_dec_hw_ctx
172 *
173 * @segment_map: Segment map buffer.
174 * @prob_tbl: Probability table buffer.
175 */
176 struct hantro_vp8_dec_hw_ctx {
177 struct hantro_aux_buf segment_map;
178 struct hantro_aux_buf prob_tbl;
179 };
180
181 /**
182 * struct hantro_vp9_frame_info
183 *
184 * @valid: frame info valid flag
185 * @frame_context_idx: index of frame context
186 * @reference_mode: inter prediction type
187 * @tx_mode: transform mode
188 * @interpolation_filter: filter selection for inter prediction
189 * @flags: frame flags
190 * @timestamp: frame timestamp
191 */
192 struct hantro_vp9_frame_info {
193 u32 valid : 1;
194 u32 frame_context_idx : 2;
195 u32 reference_mode : 2;
196 u32 tx_mode : 3;
197 u32 interpolation_filter : 3;
198 u32 flags;
199 u64 timestamp;
200 };
201
202 #define MAX_SB_COLS 64
203 #define MAX_SB_ROWS 34
204
205 /**
206 * struct hantro_vp9_dec_hw_ctx
207 *
208 * @tile_edge: auxiliary DMA buffer for tile edge processing
209 * @segment_map: auxiliary DMA buffer for segment map
210 * @misc: auxiliary DMA buffer for tile info, probabilities and hw counters
211 * @cnts: vp9 library struct for abstracting hw counters access
212 * @probability_tables: VP9 probability tables implied by the spec
213 * @frame_context: VP9 frame contexts
214 * @cur: current frame information
215 * @last: last frame information
216 * @bsd_ctrl_offset: bsd offset into tile_edge
217 * @segment_map_size: size of segment map
218 * @ctx_counters_offset: hw counters offset into misc
219 * @tile_info_offset: tile info offset into misc
220 * @tile_r_info: per-tile information array
221 * @tile_c_info: per-tile information array
222 * @last_tile_r: last number of tile rows
223 * @last_tile_c: last number of tile cols
224 * @last_sbs_r: last number of superblock rows
225 * @last_sbs_c: last number of superblock cols
226 * @active_segment: number of active segment (alternating between 0 and 1)
227 * @feature_enabled: segmentation feature enabled flags
228 * @feature_data: segmentation feature data
229 */
230 struct hantro_vp9_dec_hw_ctx {
231 struct hantro_aux_buf tile_edge;
232 struct hantro_aux_buf segment_map;
233 struct hantro_aux_buf misc;
234 struct v4l2_vp9_frame_symbol_counts cnts;
235 struct v4l2_vp9_frame_context probability_tables;
236 struct v4l2_vp9_frame_context frame_context[4];
237 struct hantro_vp9_frame_info cur;
238 struct hantro_vp9_frame_info last;
239
240 unsigned int bsd_ctrl_offset;
241 unsigned int segment_map_size;
242 unsigned int ctx_counters_offset;
243 unsigned int tile_info_offset;
244
245 unsigned short tile_r_info[MAX_SB_ROWS];
246 unsigned short tile_c_info[MAX_SB_COLS];
247 unsigned int last_tile_r;
248 unsigned int last_tile_c;
249 unsigned int last_sbs_r;
250 unsigned int last_sbs_c;
251
252 unsigned int active_segment;
253 u8 feature_enabled[8];
254 s16 feature_data[8][4];
255 };
256
257 /**
258 * struct hantro_av1_dec_ctrls
259 * @sequence: AV1 Sequence
260 * @tile_group_entry: AV1 Tile Group entry
261 * @frame: AV1 Frame Header OBU
262 * @film_grain: AV1 Film Grain
263 */
264 struct hantro_av1_dec_ctrls {
265 const struct v4l2_ctrl_av1_sequence *sequence;
266 const struct v4l2_ctrl_av1_tile_group_entry *tile_group_entry;
267 const struct v4l2_ctrl_av1_frame *frame;
268 const struct v4l2_ctrl_av1_film_grain *film_grain;
269 };
270
271 struct hantro_av1_frame_ref {
272 int width;
273 int height;
274 int mi_cols;
275 int mi_rows;
276 u64 timestamp;
277 enum v4l2_av1_frame_type frame_type;
278 bool used;
279 u32 order_hint;
280 u32 order_hints[V4L2_AV1_TOTAL_REFS_PER_FRAME];
281 struct vb2_v4l2_buffer *vb2_ref;
282 };
283
284 /**
285 * struct hantro_av1_dec_hw_ctx
286 * @db_data_col: db tile col data buffer
287 * @db_ctrl_col: db tile col ctrl buffer
288 * @cdef_col: cdef tile col buffer
289 * @sr_col: sr tile col buffer
290 * @lr_col: lr tile col buffer
291 * @global_model: global model buffer
292 * @tile_info: tile info buffer
293 * @segment: segmentation info buffer
294 * @film_grain: film grain buffer
295 * @prob_tbl: probability table
296 * @prob_tbl_out: probability table output
297 * @tile_buf: tile buffer
298 * @ctrls: V4L2 controls attached to a run
299 * @frame_refs: reference frames info slots
300 * @ref_frame_sign_bias: array of sign bias
301 * @num_tile_cols_allocated: number of allocated tiles
302 * @cdfs: current probabilities structure
303 * @cdfs_ndvc: current mv probabilities structure
304 * @default_cdfs: default probabilities structure
305 * @default_cdfs_ndvc: default mv probabilties structure
306 * @cdfs_last: stored probabilities structures
307 * @cdfs_last_ndvc: stored mv probabilities structures
308 * @current_frame_index: index of the current in frame_refs array
309 */
310 struct hantro_av1_dec_hw_ctx {
311 struct hantro_aux_buf db_data_col;
312 struct hantro_aux_buf db_ctrl_col;
313 struct hantro_aux_buf cdef_col;
314 struct hantro_aux_buf sr_col;
315 struct hantro_aux_buf lr_col;
316 struct hantro_aux_buf global_model;
317 struct hantro_aux_buf tile_info;
318 struct hantro_aux_buf segment;
319 struct hantro_aux_buf film_grain;
320 struct hantro_aux_buf prob_tbl;
321 struct hantro_aux_buf prob_tbl_out;
322 struct hantro_aux_buf tile_buf;
323 struct hantro_av1_dec_ctrls ctrls;
324 struct hantro_av1_frame_ref frame_refs[AV1_MAX_FRAME_BUF_COUNT];
325 u32 ref_frame_sign_bias[V4L2_AV1_TOTAL_REFS_PER_FRAME];
326 unsigned int num_tile_cols_allocated;
327 struct av1cdfs *cdfs;
328 struct mvcdfs *cdfs_ndvc;
329 struct av1cdfs default_cdfs;
330 struct mvcdfs default_cdfs_ndvc;
331 struct av1cdfs cdfs_last[NUM_REF_FRAMES];
332 struct mvcdfs cdfs_last_ndvc[NUM_REF_FRAMES];
333 int current_frame_index;
334 };
335 /**
336 * struct hantro_postproc_ctx
337 *
338 * @dec_q: References buffers, in decoder format.
339 */
340 struct hantro_postproc_ctx {
341 struct hantro_aux_buf dec_q[MAX_POSTPROC_BUFFERS];
342 };
343
344 /**
345 * struct hantro_postproc_ops - post-processor operations
346 *
347 * @enable: Enable the post-processor block. Optional.
348 * @disable: Disable the post-processor block. Optional.
349 * @enum_framesizes: Enumerate possible scaled output formats.
350 * Returns zero if OK, a negative value in error cases.
351 * Optional.
352 */
353 struct hantro_postproc_ops {
354 void (*enable)(struct hantro_ctx *ctx);
355 void (*disable)(struct hantro_ctx *ctx);
356 int (*enum_framesizes)(struct hantro_ctx *ctx, struct v4l2_frmsizeenum *fsize);
357 };
358
359 /**
360 * struct hantro_codec_ops - codec mode specific operations
361 *
362 * @init: If needed, can be used for initialization.
363 * Optional and called from process context.
364 * @exit: If needed, can be used to undo the .init phase.
365 * Optional and called from process context.
366 * @run: Start single {en,de)coding job. Called from atomic context
367 * to indicate that a pair of buffers is ready and the hardware
368 * should be programmed and started. Returns zero if OK, a
369 * negative value in error cases.
370 * @done: Read back processing results and additional data from hardware.
371 * @reset: Reset the hardware in case of a timeout.
372 */
373 struct hantro_codec_ops {
374 int (*init)(struct hantro_ctx *ctx);
375 void (*exit)(struct hantro_ctx *ctx);
376 int (*run)(struct hantro_ctx *ctx);
377 void (*done)(struct hantro_ctx *ctx);
378 void (*reset)(struct hantro_ctx *ctx);
379 };
380
381 /**
382 * enum hantro_enc_fmt - source format ID for hardware registers.
383 *
384 * @ROCKCHIP_VPU_ENC_FMT_YUV420P: Y/CbCr 4:2:0 planar format
385 * @ROCKCHIP_VPU_ENC_FMT_YUV420SP: Y/CbCr 4:2:0 semi-planar format
386 * @ROCKCHIP_VPU_ENC_FMT_YUYV422: YUV 4:2:2 packed format (YUYV)
387 * @ROCKCHIP_VPU_ENC_FMT_UYVY422: YUV 4:2:2 packed format (UYVY)
388 */
389 enum hantro_enc_fmt {
390 ROCKCHIP_VPU_ENC_FMT_YUV420P = 0,
391 ROCKCHIP_VPU_ENC_FMT_YUV420SP = 1,
392 ROCKCHIP_VPU_ENC_FMT_YUYV422 = 2,
393 ROCKCHIP_VPU_ENC_FMT_UYVY422 = 3,
394 };
395
396 extern const struct hantro_variant imx8mm_vpu_g1_variant;
397 extern const struct hantro_variant imx8mq_vpu_g1_variant;
398 extern const struct hantro_variant imx8mq_vpu_g2_variant;
399 extern const struct hantro_variant imx8mq_vpu_variant;
400 extern const struct hantro_variant px30_vpu_variant;
401 extern const struct hantro_variant rk3036_vpu_variant;
402 extern const struct hantro_variant rk3066_vpu_variant;
403 extern const struct hantro_variant rk3288_vpu_variant;
404 extern const struct hantro_variant rk3328_vpu_variant;
405 extern const struct hantro_variant rk3399_vpu_variant;
406 extern const struct hantro_variant rk3568_vepu_variant;
407 extern const struct hantro_variant rk3568_vpu_variant;
408 extern const struct hantro_variant rk3588_vpu981_variant;
409 extern const struct hantro_variant sama5d4_vdec_variant;
410 extern const struct hantro_variant sunxi_vpu_variant;
411 extern const struct hantro_variant stm32mp25_vdec_variant;
412 extern const struct hantro_variant stm32mp25_venc_variant;
413
414 extern const struct hantro_postproc_ops hantro_g1_postproc_ops;
415 extern const struct hantro_postproc_ops hantro_g2_postproc_ops;
416 extern const struct hantro_postproc_ops rockchip_vpu981_postproc_ops;
417
418 extern const u32 hantro_vp8_dec_mc_filter[8][6];
419
420 void hantro_watchdog(struct work_struct *work);
421 void hantro_run(struct hantro_ctx *ctx);
422 void hantro_irq_done(struct hantro_dev *vpu,
423 enum vb2_buffer_state result);
424 void hantro_start_prepare_run(struct hantro_ctx *ctx);
425 void hantro_end_prepare_run(struct hantro_ctx *ctx);
426
427 irqreturn_t hantro_g1_irq(int irq, void *dev_id);
428 void hantro_g1_reset(struct hantro_ctx *ctx);
429
430 int hantro_h1_jpeg_enc_run(struct hantro_ctx *ctx);
431 int rockchip_vpu2_jpeg_enc_run(struct hantro_ctx *ctx);
432 void hantro_h1_jpeg_enc_done(struct hantro_ctx *ctx);
433 void rockchip_vpu2_jpeg_enc_done(struct hantro_ctx *ctx);
434
435 dma_addr_t hantro_h264_get_ref_buf(struct hantro_ctx *ctx,
436 unsigned int dpb_idx);
437 u16 hantro_h264_get_ref_nbr(struct hantro_ctx *ctx,
438 unsigned int dpb_idx);
439 int hantro_h264_dec_prepare_run(struct hantro_ctx *ctx);
440 int rockchip_vpu2_h264_dec_run(struct hantro_ctx *ctx);
441 int hantro_g1_h264_dec_run(struct hantro_ctx *ctx);
442 int hantro_h264_dec_init(struct hantro_ctx *ctx);
443 void hantro_h264_dec_exit(struct hantro_ctx *ctx);
444
445 int hantro_hevc_dec_init(struct hantro_ctx *ctx);
446 void hantro_hevc_dec_exit(struct hantro_ctx *ctx);
447 int hantro_g2_hevc_dec_run(struct hantro_ctx *ctx);
448 int hantro_hevc_dec_prepare_run(struct hantro_ctx *ctx);
449 void hantro_hevc_ref_init(struct hantro_ctx *ctx);
450 dma_addr_t hantro_hevc_get_ref_buf(struct hantro_ctx *ctx, s32 poc);
451 int hantro_hevc_add_ref_buf(struct hantro_ctx *ctx, int poc, dma_addr_t addr);
452
453 int rockchip_vpu981_av1_dec_init(struct hantro_ctx *ctx);
454 void rockchip_vpu981_av1_dec_exit(struct hantro_ctx *ctx);
455 int rockchip_vpu981_av1_dec_run(struct hantro_ctx *ctx);
456 void rockchip_vpu981_av1_dec_done(struct hantro_ctx *ctx);
457
hantro_vp9_num_sbs(unsigned short dimension)458 static inline unsigned short hantro_vp9_num_sbs(unsigned short dimension)
459 {
460 return (dimension + 63) / 64;
461 }
462
463 static inline size_t
hantro_vp9_mv_size(unsigned int width,unsigned int height)464 hantro_vp9_mv_size(unsigned int width, unsigned int height)
465 {
466 int num_ctbs;
467
468 /*
469 * There can be up to (CTBs x 64) number of blocks,
470 * and the motion vector for each block needs 16 bytes.
471 */
472 num_ctbs = hantro_vp9_num_sbs(width) * hantro_vp9_num_sbs(height);
473 return (num_ctbs * 64) * 16;
474 }
475
476 static inline size_t
hantro_h264_mv_size(unsigned int width,unsigned int height)477 hantro_h264_mv_size(unsigned int width, unsigned int height)
478 {
479 /*
480 * A decoded 8-bit 4:2:0 NV12 frame may need memory for up to
481 * 448 bytes per macroblock with additional 32 bytes on
482 * multi-core variants.
483 *
484 * The H264 decoder needs extra space on the output buffers
485 * to store motion vectors. This is needed for reference
486 * frames and only if the format is non-post-processed NV12.
487 *
488 * Memory layout is as follow:
489 *
490 * +---------------------------+
491 * | Y-plane 256 bytes x MBs |
492 * +---------------------------+
493 * | UV-plane 128 bytes x MBs |
494 * +---------------------------+
495 * | MV buffer 64 bytes x MBs |
496 * +---------------------------+
497 * | MC sync 32 bytes |
498 * +---------------------------+
499 */
500 return 64 * MB_WIDTH(width) * MB_WIDTH(height) + 32;
501 }
502
503 static inline size_t
hantro_hevc_mv_size(unsigned int width,unsigned int height)504 hantro_hevc_mv_size(unsigned int width, unsigned int height)
505 {
506 /*
507 * A CTB can be 64x64, 32x32 or 16x16.
508 * Allocated memory for the "worse" case: 16x16
509 */
510 return width * height / 16;
511 }
512
hantro_av1_num_sbs(unsigned short dimension)513 static inline unsigned short hantro_av1_num_sbs(unsigned short dimension)
514 {
515 return DIV_ROUND_UP(dimension, 64);
516 }
517
518 static inline size_t
hantro_av1_mv_size(unsigned int width,unsigned int height)519 hantro_av1_mv_size(unsigned int width, unsigned int height)
520 {
521 size_t num_sbs = hantro_av1_num_sbs(width) * hantro_av1_num_sbs(height);
522
523 return ALIGN(num_sbs * 384, 16) * 2 + 512;
524 }
525
526 size_t hantro_g2_chroma_offset(struct hantro_ctx *ctx);
527 size_t hantro_g2_motion_vectors_offset(struct hantro_ctx *ctx);
528
529 int hantro_g1_mpeg2_dec_run(struct hantro_ctx *ctx);
530 int rockchip_vpu2_mpeg2_dec_run(struct hantro_ctx *ctx);
531 void hantro_mpeg2_dec_copy_qtable(u8 *qtable,
532 const struct v4l2_ctrl_mpeg2_quantisation *ctrl);
533 int hantro_mpeg2_dec_init(struct hantro_ctx *ctx);
534 void hantro_mpeg2_dec_exit(struct hantro_ctx *ctx);
535
536 int hantro_g1_vp8_dec_run(struct hantro_ctx *ctx);
537 int rockchip_vpu2_vp8_dec_run(struct hantro_ctx *ctx);
538 int hantro_vp8_dec_init(struct hantro_ctx *ctx);
539 void hantro_vp8_dec_exit(struct hantro_ctx *ctx);
540 void hantro_vp8_prob_update(struct hantro_ctx *ctx,
541 const struct v4l2_ctrl_vp8_frame *hdr);
542
543 int hantro_g2_vp9_dec_run(struct hantro_ctx *ctx);
544 void hantro_g2_vp9_dec_done(struct hantro_ctx *ctx);
545 int hantro_vp9_dec_init(struct hantro_ctx *ctx);
546 void hantro_vp9_dec_exit(struct hantro_ctx *ctx);
547 void hantro_g2_check_idle(struct hantro_dev *vpu);
548 irqreturn_t hantro_g2_irq(int irq, void *dev_id);
549
550 #endif /* HANTRO_HW_H_ */
551