2 * AV1 HW decode acceleration through VA API
4 * This file is part of FFmpeg.
6 * FFmpeg is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * FFmpeg is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with FFmpeg; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 #include "libavutil/frame.h"
22 #include "hwaccel_internal.h"
23 #include "vaapi_decode.h"
28 typedef struct VAAPIAV1FrameRef
{
33 typedef struct VAAPIAV1DecContext
{
34 VAAPIDecodeContext base
;
37 * For film grain case, VAAPI generate 2 output for each frame,
38 * current_frame will not apply film grain, and will be used for
39 * references for next frames. Maintain the reference list without
40 * applying film grain here. And current_display_picture will be
41 * used to apply film grain and push to downstream.
43 VAAPIAV1FrameRef ref_tab
[AV1_NUM_REF_FRAMES
];
47 static VASurfaceID
vaapi_av1_surface_id(AV1Frame
*vf
)
50 return ff_vaapi_get_surface_id(vf
->f
);
52 return VA_INVALID_SURFACE
;
55 static int8_t vaapi_av1_get_bit_depth_idx(AVCodecContext
*avctx
)
57 AV1DecContext
*s
= avctx
->priv_data
;
58 const AV1RawSequenceHeader
*seq
= s
->raw_seq
;
61 if (seq
->seq_profile
== 2 && seq
->color_config
.high_bitdepth
)
62 bit_depth
= seq
->color_config
.twelve_bit
? 12 : 10;
63 else if (seq
->seq_profile
<= 2)
64 bit_depth
= seq
->color_config
.high_bitdepth
? 10 : 8;
66 av_log(avctx
, AV_LOG_ERROR
,
67 "Couldn't get bit depth from profile:%d.\n", seq
->seq_profile
);
70 return bit_depth
== 8 ? 0 : bit_depth
== 10 ? 1 : 2;
73 static int vaapi_av1_decode_init(AVCodecContext
*avctx
)
75 VAAPIAV1DecContext
*ctx
= avctx
->internal
->hwaccel_priv_data
;
77 ctx
->tmp_frame
= av_frame_alloc();
79 return AVERROR(ENOMEM
);
81 for (int i
= 0; i
< FF_ARRAY_ELEMS(ctx
->ref_tab
); i
++) {
82 ctx
->ref_tab
[i
].frame
= av_frame_alloc();
83 if (!ctx
->ref_tab
[i
].frame
)
84 return AVERROR(ENOMEM
);
85 ctx
->ref_tab
[i
].valid
= 0;
88 return ff_vaapi_decode_init(avctx
);
91 static int vaapi_av1_decode_uninit(AVCodecContext
*avctx
)
93 VAAPIAV1DecContext
*ctx
= avctx
->internal
->hwaccel_priv_data
;
95 av_frame_free(&ctx
->tmp_frame
);
97 for (int i
= 0; i
< FF_ARRAY_ELEMS(ctx
->ref_tab
); i
++)
98 av_frame_free(&ctx
->ref_tab
[i
].frame
);
100 return ff_vaapi_decode_uninit(avctx
);
104 static int vaapi_av1_start_frame(AVCodecContext
*avctx
,
105 av_unused
const uint8_t *buffer
,
106 av_unused
uint32_t size
)
108 AV1DecContext
*s
= avctx
->priv_data
;
109 const AV1RawSequenceHeader
*seq
= s
->raw_seq
;
110 const AV1RawFrameHeader
*frame_header
= s
->raw_frame_header
;
111 const AV1RawFilmGrainParams
*film_grain
= &s
->cur_frame
.film_grain
;
112 VAAPIDecodePicture
*pic
= s
->cur_frame
.hwaccel_picture_private
;
113 VAAPIAV1DecContext
*ctx
= avctx
->internal
->hwaccel_priv_data
;
114 VADecPictureParameterBufferAV1 pic_param
;
115 int8_t bit_depth_idx
;
117 int apply_grain
= !(avctx
->export_side_data
& AV_CODEC_EXPORT_DATA_FILM_GRAIN
) && film_grain
->apply_grain
;
118 uint8_t remap_lr_type
[4] = {AV1_RESTORE_NONE
, AV1_RESTORE_SWITCHABLE
, AV1_RESTORE_WIENER
, AV1_RESTORE_SGRPROJ
};
119 uint8_t segmentation_feature_signed
[AV1_SEG_LVL_MAX
] = {1, 1, 1, 1, 1, 0, 0, 0};
120 uint8_t segmentation_feature_max
[AV1_SEG_LVL_MAX
] = {255, AV1_MAX_LOOP_FILTER
,
121 AV1_MAX_LOOP_FILTER
, AV1_MAX_LOOP_FILTER
, AV1_MAX_LOOP_FILTER
, 7 , 0 , 0 };
123 bit_depth_idx
= vaapi_av1_get_bit_depth_idx(avctx
);
124 if (bit_depth_idx
< 0)
128 if (ctx
->tmp_frame
->buf
[0])
129 av_frame_unref(ctx
->tmp_frame
);
130 err
= ff_thread_get_buffer(avctx
, ctx
->tmp_frame
, AV_GET_BUFFER_FLAG_REF
);
133 pic
->output_surface
= ff_vaapi_get_surface_id(ctx
->tmp_frame
);
135 pic
->output_surface
= vaapi_av1_surface_id(&s
->cur_frame
);
138 memset(&pic_param
, 0, sizeof(VADecPictureParameterBufferAV1
));
139 pic_param
= (VADecPictureParameterBufferAV1
) {
140 .profile
= seq
->seq_profile
,
141 .order_hint_bits_minus_1
= seq
->order_hint_bits_minus_1
,
142 .bit_depth_idx
= bit_depth_idx
,
143 .matrix_coefficients
= seq
->color_config
.matrix_coefficients
,
144 .current_frame
= pic
->output_surface
,
145 .current_display_picture
= vaapi_av1_surface_id(&s
->cur_frame
),
146 .frame_width_minus1
= frame_header
->frame_width_minus_1
,
147 .frame_height_minus1
= frame_header
->frame_height_minus_1
,
148 .primary_ref_frame
= frame_header
->primary_ref_frame
,
149 .order_hint
= frame_header
->order_hint
,
150 .tile_cols
= frame_header
->tile_cols
,
151 .tile_rows
= frame_header
->tile_rows
,
152 .context_update_tile_id
= frame_header
->context_update_tile_id
,
153 .superres_scale_denominator
= frame_header
->use_superres
?
154 frame_header
->coded_denom
+ AV1_SUPERRES_DENOM_MIN
:
156 .interp_filter
= frame_header
->interpolation_filter
,
157 .filter_level
[0] = frame_header
->loop_filter_level
[0],
158 .filter_level
[1] = frame_header
->loop_filter_level
[1],
159 .filter_level_u
= frame_header
->loop_filter_level
[2],
160 .filter_level_v
= frame_header
->loop_filter_level
[3],
161 .base_qindex
= frame_header
->base_q_idx
,
162 .y_dc_delta_q
= frame_header
->delta_q_y_dc
,
163 .u_dc_delta_q
= frame_header
->delta_q_u_dc
,
164 .u_ac_delta_q
= frame_header
->delta_q_u_ac
,
165 .v_dc_delta_q
= frame_header
->delta_q_v_dc
,
166 .v_ac_delta_q
= frame_header
->delta_q_v_ac
,
167 .cdef_damping_minus_3
= frame_header
->cdef_damping_minus_3
,
168 .cdef_bits
= frame_header
->cdef_bits
,
169 .seq_info_fields
.fields
= {
170 .still_picture
= seq
->still_picture
,
171 .use_128x128_superblock
= seq
->use_128x128_superblock
,
172 .enable_filter_intra
= seq
->enable_filter_intra
,
173 .enable_intra_edge_filter
= seq
->enable_intra_edge_filter
,
174 .enable_interintra_compound
= seq
->enable_interintra_compound
,
175 .enable_masked_compound
= seq
->enable_masked_compound
,
176 .enable_dual_filter
= seq
->enable_dual_filter
,
177 .enable_order_hint
= seq
->enable_order_hint
,
178 .enable_jnt_comp
= seq
->enable_jnt_comp
,
179 .enable_cdef
= seq
->enable_cdef
,
180 .mono_chrome
= seq
->color_config
.mono_chrome
,
181 .color_range
= seq
->color_config
.color_range
,
182 .subsampling_x
= seq
->color_config
.subsampling_x
,
183 .subsampling_y
= seq
->color_config
.subsampling_y
,
184 .chroma_sample_position
= seq
->color_config
.chroma_sample_position
,
185 .film_grain_params_present
= seq
->film_grain_params_present
&&
186 !(avctx
->export_side_data
& AV_CODEC_EXPORT_DATA_FILM_GRAIN
),
188 .seg_info
.segment_info_fields
.bits
= {
189 .enabled
= frame_header
->segmentation_enabled
,
190 .update_map
= frame_header
->segmentation_update_map
,
191 .temporal_update
= frame_header
->segmentation_temporal_update
,
192 .update_data
= frame_header
->segmentation_update_data
,
195 .film_grain_info_fields
.bits
= {
196 .apply_grain
= apply_grain
,
197 .chroma_scaling_from_luma
= film_grain
->chroma_scaling_from_luma
,
198 .grain_scaling_minus_8
= film_grain
->grain_scaling_minus_8
,
199 .ar_coeff_lag
= film_grain
->ar_coeff_lag
,
200 .ar_coeff_shift_minus_6
= film_grain
->ar_coeff_shift_minus_6
,
201 .grain_scale_shift
= film_grain
->grain_scale_shift
,
202 .overlap_flag
= film_grain
->overlap_flag
,
203 .clip_to_restricted_range
= film_grain
->clip_to_restricted_range
,
205 .grain_seed
= film_grain
->grain_seed
,
206 .num_y_points
= film_grain
->num_y_points
,
207 .num_cb_points
= film_grain
->num_cb_points
,
208 .num_cr_points
= film_grain
->num_cr_points
,
209 .cb_mult
= film_grain
->cb_mult
,
210 .cb_luma_mult
= film_grain
->cb_luma_mult
,
211 .cb_offset
= film_grain
->cb_offset
,
212 .cr_mult
= film_grain
->cr_mult
,
213 .cr_luma_mult
= film_grain
->cr_luma_mult
,
214 .cr_offset
= film_grain
->cr_offset
,
216 .pic_info_fields
.bits
= {
217 .frame_type
= frame_header
->frame_type
,
218 .show_frame
= frame_header
->show_frame
,
219 .showable_frame
= frame_header
->showable_frame
,
220 .error_resilient_mode
= frame_header
->error_resilient_mode
,
221 .disable_cdf_update
= frame_header
->disable_cdf_update
,
222 .allow_screen_content_tools
= frame_header
->allow_screen_content_tools
,
223 .force_integer_mv
= frame_header
->force_integer_mv
,
224 .allow_intrabc
= frame_header
->allow_intrabc
,
225 .use_superres
= frame_header
->use_superres
,
226 .allow_high_precision_mv
= frame_header
->allow_high_precision_mv
,
227 .is_motion_mode_switchable
= frame_header
->is_motion_mode_switchable
,
228 .use_ref_frame_mvs
= frame_header
->use_ref_frame_mvs
,
229 .disable_frame_end_update_cdf
= frame_header
->disable_frame_end_update_cdf
,
230 .uniform_tile_spacing_flag
= frame_header
->uniform_tile_spacing_flag
,
231 .allow_warped_motion
= frame_header
->allow_warped_motion
,
233 .loop_filter_info_fields
.bits
= {
234 .sharpness_level
= frame_header
->loop_filter_sharpness
,
235 .mode_ref_delta_enabled
= frame_header
->loop_filter_delta_enabled
,
236 .mode_ref_delta_update
= frame_header
->loop_filter_delta_update
,
238 .mode_control_fields
.bits
= {
239 .delta_q_present_flag
= frame_header
->delta_q_present
,
240 .log2_delta_q_res
= frame_header
->delta_q_res
,
241 .delta_lf_present_flag
= frame_header
->delta_lf_present
,
242 .log2_delta_lf_res
= frame_header
->delta_lf_res
,
243 .delta_lf_multi
= frame_header
->delta_lf_multi
,
244 .tx_mode
= frame_header
->tx_mode
,
245 .reference_select
= frame_header
->reference_select
,
246 .reduced_tx_set_used
= frame_header
->reduced_tx_set
,
247 .skip_mode_present
= frame_header
->skip_mode_present
,
249 .loop_restoration_fields
.bits
= {
250 .yframe_restoration_type
= remap_lr_type
[frame_header
->lr_type
[0]],
251 .cbframe_restoration_type
= remap_lr_type
[frame_header
->lr_type
[1]],
252 .crframe_restoration_type
= remap_lr_type
[frame_header
->lr_type
[2]],
253 .lr_unit_shift
= frame_header
->lr_unit_shift
,
254 .lr_uv_shift
= frame_header
->lr_uv_shift
,
256 .qmatrix_fields
.bits
= {
257 .using_qmatrix
= frame_header
->using_qmatrix
,
258 .qm_y
= frame_header
->qm_y
,
259 .qm_u
= frame_header
->qm_u
,
260 .qm_v
= frame_header
->qm_v
,
264 for (int i
= 0; i
< AV1_NUM_REF_FRAMES
; i
++) {
265 if (pic_param
.pic_info_fields
.bits
.frame_type
== AV1_FRAME_KEY
&& frame_header
->show_frame
)
266 pic_param
.ref_frame_map
[i
] = VA_INVALID_ID
;
268 pic_param
.ref_frame_map
[i
] = ctx
->ref_tab
[i
].valid
?
269 ff_vaapi_get_surface_id(ctx
->ref_tab
[i
].frame
) :
270 vaapi_av1_surface_id(&s
->ref
[i
]);
272 for (int i
= 0; i
< AV1_REFS_PER_FRAME
; i
++) {
273 pic_param
.ref_frame_idx
[i
] = frame_header
->ref_frame_idx
[i
];
275 for (int i
= 0; i
< AV1_TOTAL_REFS_PER_FRAME
; i
++) {
276 pic_param
.ref_deltas
[i
] = frame_header
->loop_filter_ref_deltas
[i
];
278 for (int i
= 0; i
< 2; i
++) {
279 pic_param
.mode_deltas
[i
] = frame_header
->loop_filter_mode_deltas
[i
];
281 for (int i
= 0; i
< (1 << frame_header
->cdef_bits
); i
++) {
282 pic_param
.cdef_y_strengths
[i
] =
283 (frame_header
->cdef_y_pri_strength
[i
] << 2) +
284 frame_header
->cdef_y_sec_strength
[i
];
285 pic_param
.cdef_uv_strengths
[i
] =
286 (frame_header
->cdef_uv_pri_strength
[i
] << 2) +
287 frame_header
->cdef_uv_sec_strength
[i
];
289 for (int i
= 0; i
< frame_header
->tile_cols
; i
++) {
290 pic_param
.width_in_sbs_minus_1
[i
] =
291 frame_header
->width_in_sbs_minus_1
[i
];
293 for (int i
= 0; i
< frame_header
->tile_rows
; i
++) {
294 pic_param
.height_in_sbs_minus_1
[i
] =
295 frame_header
->height_in_sbs_minus_1
[i
];
297 for (int i
= AV1_REF_FRAME_LAST
; i
<= AV1_REF_FRAME_ALTREF
; i
++) {
298 pic_param
.wm
[i
- 1].invalid
= s
->cur_frame
.gm_invalid
[i
];
299 pic_param
.wm
[i
- 1].wmtype
= s
->cur_frame
.gm_type
[i
];
300 for (int j
= 0; j
< 6; j
++)
301 pic_param
.wm
[i
- 1].wmmat
[j
] = s
->cur_frame
.gm_params
[i
][j
];
303 for (int i
= 0; i
< AV1_MAX_SEGMENTS
; i
++) {
304 for (int j
= 0; j
< AV1_SEG_LVL_MAX
; j
++) {
305 pic_param
.seg_info
.feature_mask
[i
] |= (frame_header
->feature_enabled
[i
][j
] << j
);
306 if (segmentation_feature_signed
[j
])
307 pic_param
.seg_info
.feature_data
[i
][j
] = av_clip(frame_header
->feature_value
[i
][j
],
308 -segmentation_feature_max
[j
], segmentation_feature_max
[j
]);
310 pic_param
.seg_info
.feature_data
[i
][j
] = av_clip(frame_header
->feature_value
[i
][j
],
311 0, segmentation_feature_max
[j
]);
315 for (int i
= 0; i
< film_grain
->num_y_points
; i
++) {
316 pic_param
.film_grain_info
.point_y_value
[i
] =
317 film_grain
->point_y_value
[i
];
318 pic_param
.film_grain_info
.point_y_scaling
[i
] =
319 film_grain
->point_y_scaling
[i
];
321 for (int i
= 0; i
< film_grain
->num_cb_points
; i
++) {
322 pic_param
.film_grain_info
.point_cb_value
[i
] =
323 film_grain
->point_cb_value
[i
];
324 pic_param
.film_grain_info
.point_cb_scaling
[i
] =
325 film_grain
->point_cb_scaling
[i
];
327 for (int i
= 0; i
< film_grain
->num_cr_points
; i
++) {
328 pic_param
.film_grain_info
.point_cr_value
[i
] =
329 film_grain
->point_cr_value
[i
];
330 pic_param
.film_grain_info
.point_cr_scaling
[i
] =
331 film_grain
->point_cr_scaling
[i
];
333 for (int i
= 0; i
< 24; i
++) {
334 pic_param
.film_grain_info
.ar_coeffs_y
[i
] =
335 film_grain
->ar_coeffs_y_plus_128
[i
] - 128;
337 for (int i
= 0; i
< 25; i
++) {
338 pic_param
.film_grain_info
.ar_coeffs_cb
[i
] =
339 film_grain
->ar_coeffs_cb_plus_128
[i
] - 128;
340 pic_param
.film_grain_info
.ar_coeffs_cr
[i
] =
341 film_grain
->ar_coeffs_cr_plus_128
[i
] - 128;
344 err
= ff_vaapi_decode_make_param_buffer(avctx
, pic
,
345 VAPictureParameterBufferType
,
346 &pic_param
, sizeof(pic_param
));
353 ff_vaapi_decode_cancel(avctx
, pic
);
357 static int vaapi_av1_end_frame(AVCodecContext
*avctx
)
359 const AV1DecContext
*s
= avctx
->priv_data
;
360 const AV1RawFrameHeader
*header
= s
->raw_frame_header
;
361 const AV1RawFilmGrainParams
*film_grain
= &s
->cur_frame
.film_grain
;
362 VAAPIDecodePicture
*pic
= s
->cur_frame
.hwaccel_picture_private
;
363 VAAPIAV1DecContext
*ctx
= avctx
->internal
->hwaccel_priv_data
;
365 int apply_grain
= !(avctx
->export_side_data
& AV_CODEC_EXPORT_DATA_FILM_GRAIN
) && film_grain
->apply_grain
;
367 ret
= ff_vaapi_decode_issue(avctx
, pic
);
371 for (int i
= 0; i
< AV1_NUM_REF_FRAMES
; i
++) {
372 if (header
->refresh_frame_flags
& (1 << i
)) {
373 if (ctx
->ref_tab
[i
].frame
->buf
[0])
374 av_frame_unref(ctx
->ref_tab
[i
].frame
);
377 ret
= av_frame_ref(ctx
->ref_tab
[i
].frame
, ctx
->tmp_frame
);
380 ctx
->ref_tab
[i
].valid
= 1;
382 ctx
->ref_tab
[i
].valid
= 0;
390 static int vaapi_av1_decode_slice(AVCodecContext
*avctx
,
391 const uint8_t *buffer
,
394 const AV1DecContext
*s
= avctx
->priv_data
;
395 VAAPIDecodePicture
*pic
= s
->cur_frame
.hwaccel_picture_private
;
396 VASliceParameterBufferAV1 slice_param
;
399 for (int i
= s
->tg_start
; i
<= s
->tg_end
; i
++) {
400 memset(&slice_param
, 0, sizeof(VASliceParameterBufferAV1
));
402 slice_param
= (VASliceParameterBufferAV1
) {
403 .slice_data_size
= s
->tile_group_info
[i
].tile_size
,
404 .slice_data_offset
= s
->tile_group_info
[i
].tile_offset
,
405 .slice_data_flag
= VA_SLICE_DATA_FLAG_ALL
,
406 .tile_row
= s
->tile_group_info
[i
].tile_row
,
407 .tile_column
= s
->tile_group_info
[i
].tile_column
,
408 .tg_start
= s
->tg_start
,
412 err
= ff_vaapi_decode_make_slice_buffer(avctx
, pic
, &slice_param
,
413 sizeof(VASliceParameterBufferAV1
),
417 ff_vaapi_decode_cancel(avctx
, pic
);
425 const FFHWAccel ff_av1_vaapi_hwaccel
= {
426 .p
.name
= "av1_vaapi",
427 .p
.type
= AVMEDIA_TYPE_VIDEO
,
428 .p
.id
= AV_CODEC_ID_AV1
,
429 .p
.pix_fmt
= AV_PIX_FMT_VAAPI
,
430 .start_frame
= vaapi_av1_start_frame
,
431 .end_frame
= vaapi_av1_end_frame
,
432 .decode_slice
= vaapi_av1_decode_slice
,
433 .frame_priv_data_size
= sizeof(VAAPIDecodePicture
),
434 .init
= vaapi_av1_decode_init
,
435 .uninit
= vaapi_av1_decode_uninit
,
436 .frame_params
= ff_vaapi_common_frame_params
,
437 .priv_data_size
= sizeof(VAAPIAV1DecContext
),
438 .caps_internal
= HWACCEL_CAP_ASYNC_SAFE
,