demux: heif: refactor pic setup
[vlc.git] / modules / demux / mp4 / heif.c
blobcb3be7de11a0df08d747c0be5314df903c941f30
1 /*****************************************************************************
2 * heif.c : ISO/IEC 23008-12 HEIF still picture demuxer
3 *****************************************************************************
4 * Copyright (C) 2018 Videolabs, VLC authors and VideoLAN
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU Lesser General Public License as published by
8 * the Free Software Foundation; either version 2.1 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public License
17 * along with this program; if not, write to the Free Software Foundation,
18 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
19 *****************************************************************************/
20 #ifdef HAVE_CONFIG_H
21 # include "config.h"
22 #endif
24 /*****************************************************************************
25 * Preamble
26 *****************************************************************************/
27 #include <vlc_common.h>
28 #include <vlc_demux.h>
29 #include <vlc_input.h>
30 #include <assert.h>
31 #include <limits.h>
33 #include "libmp4.h"
34 #include "heif.h"
35 #include "../../packetizer/iso_color_tables.h"
37 struct heif_private_t
39 MP4_Box_t *p_root;
40 es_out_id_t *id;
41 vlc_tick_t i_pcr;
42 vlc_tick_t i_end_display_time;
43 vlc_tick_t i_image_duration;
44 bool b_seekpoint_changed;
45 uint32_t i_seekpoint;
46 input_title_t *p_title;
48 struct
50 MP4_Box_t *p_infe;
51 es_format_t fmt;
52 const MP4_Box_t *p_shared_header;
53 } current;
56 static MP4_Box_t * NextAtom( MP4_Box_t *p_root,
57 vlc_fourcc_t i_type, const char *psz_path,
58 MP4_Box_t *p_infe )
60 if( p_infe == NULL )
61 p_infe = MP4_BoxGet( p_root, psz_path );
62 else
63 p_infe = p_infe->p_next;
64 for( ; p_infe; p_infe = p_infe->p_next )
66 if( p_infe->i_type == i_type )
67 return p_infe;
69 return NULL;
72 static MP4_Box_t * GetAtom( MP4_Box_t *p_root, MP4_Box_t *p_atom,
73 vlc_fourcc_t i_type, const char *psz_path,
74 bool(*pf_match)(const MP4_Box_t *, void *),
75 void *priv )
77 while( (p_atom = NextAtom( p_root, i_type, psz_path, p_atom )) )
79 if( pf_match( p_atom, priv ) )
80 return p_atom;
82 return NULL;
85 static bool MatchInfeID( const MP4_Box_t *p_infe, void *priv )
87 return BOXDATA(p_infe)->i_item_id == *((uint32_t *) priv);
90 static bool MatchPureImage( const MP4_Box_t *p_infe, void *priv )
92 MP4_Box_t *p_root = priv;
93 const MP4_Box_t *p_iref = MP4_BoxGet( p_root, "meta/iref" );
94 if( !p_iref )
95 return true;
96 for( const MP4_Box_t *p_refbox = p_iref->p_first;
97 p_refbox; p_refbox = p_refbox->p_next )
99 if( BOXDATA(p_refbox)->i_from_item_id == BOXDATA(p_infe)->i_item_id )
100 return false;
102 return true;
105 static void SeekToPrevImageEnd( struct heif_private_t *p_sys, int i_picture )
107 int i = 0;
108 MP4_Box_t *p_infe = NULL;
109 while( i < i_picture &&
110 (p_infe = NextAtom( p_sys->p_root, ATOM_infe, "meta/iinf/infe", p_infe )) )
112 if( (BOXDATA(p_infe)->i_flags & 0x01) != 0x00 ||
113 !MatchPureImage( p_infe, p_sys->p_root ) )
114 continue;
115 i++;
117 p_sys->current.p_infe = p_infe;
118 p_sys->i_end_display_time = 0;
119 p_sys->i_pcr = i * p_sys->i_image_duration;
122 static int ControlHEIF( demux_t *p_demux, int i_query, va_list args )
124 struct heif_private_t *p_sys = (void *) p_demux->p_sys;
126 switch( i_query )
128 case DEMUX_CAN_SEEK:
129 *va_arg(args, bool *) = true;
130 return VLC_SUCCESS;
131 case DEMUX_GET_TITLE_INFO:
133 input_title_t ***ppp_title = va_arg( args, input_title_t *** );
134 int *pi_int = va_arg( args, int* );
135 int *pi_title_offset = va_arg( args, int* );
136 int *pi_seekpoint_offset = va_arg( args, int* );
138 if( !p_sys->p_title )
139 return VLC_EGENERIC;
141 *pi_int = 1;
142 *ppp_title = malloc( sizeof( input_title_t*) );
143 (*ppp_title)[0] = vlc_input_title_Duplicate( p_sys->p_title );
144 *pi_title_offset = 0;
145 *pi_seekpoint_offset = 0;
146 return VLC_SUCCESS;
148 case DEMUX_SET_TITLE:
150 const int i_title = va_arg( args, int );
151 if( !p_sys->p_title || i_title != 0 )
152 return VLC_EGENERIC;
153 return VLC_SUCCESS;
155 case DEMUX_GET_SEEKPOINT:
156 *va_arg( args, int * ) = p_sys->i_seekpoint;
157 return VLC_SUCCESS;
158 case DEMUX_SET_SEEKPOINT:
160 const int i_seekpoint = va_arg( args, int );
161 if( !p_sys->p_title )
162 return VLC_EGENERIC;
163 SeekToPrevImageEnd( p_sys, i_seekpoint );
164 return VLC_SUCCESS;
166 case DEMUX_TEST_AND_CLEAR_FLAGS:
168 unsigned *restrict flags = va_arg( args, unsigned * );
170 if ((*flags & INPUT_UPDATE_SEEKPOINT) && p_sys->b_seekpoint_changed)
172 *flags = INPUT_UPDATE_SEEKPOINT;
173 p_sys->b_seekpoint_changed = false;
175 else
176 *flags = 0;
177 return VLC_SUCCESS;
179 case DEMUX_GET_LENGTH:
180 *(va_arg( args, vlc_tick_t * )) = p_sys->p_title->i_seekpoint *
181 p_sys->i_image_duration;
182 return VLC_SUCCESS;
183 case DEMUX_GET_TIME:
184 *(va_arg(args, vlc_tick_t *)) = p_sys->i_pcr;
185 return VLC_SUCCESS;
186 case DEMUX_SET_TIME:
188 SeekToPrevImageEnd( p_sys, va_arg(args, vlc_tick_t) /
189 p_sys->i_image_duration );
190 return VLC_SUCCESS;
192 case DEMUX_GET_POSITION:
193 if( !p_sys->p_title->i_seekpoint )
194 return VLC_EGENERIC;
195 *(va_arg(args, double *)) = (double) p_sys->i_pcr /
196 (p_sys->p_title->i_seekpoint * p_sys->i_image_duration);
197 return VLC_SUCCESS;
198 case DEMUX_SET_POSITION:
200 SeekToPrevImageEnd( p_sys, va_arg(args, double) * p_sys->p_title->i_seekpoint );
201 return VLC_SUCCESS;
203 case DEMUX_CAN_PAUSE:
204 case DEMUX_SET_PAUSE_STATE:
205 case DEMUX_CAN_CONTROL_PACE:
206 case DEMUX_GET_PTS_DELAY:
207 return demux_vaControlHelper( p_demux->s, 0, -1, 0, 1, i_query, args );
209 default:
210 return VLC_EGENERIC;
215 //static int DemuxCompositeImage( demux_t *p_demux )
220 static block_t *ReadItemExtents( demux_t *p_demux, uint32_t i_item_id,
221 const MP4_Box_t *p_shared_header )
223 struct heif_private_t *p_sys = (void *) p_demux->p_sys;
224 block_t *p_block = NULL;
226 MP4_Box_t *p_iloc = MP4_BoxGet( p_sys->p_root, "meta/iloc" );
227 if( !p_iloc )
228 return p_block;
230 for( uint32_t i=0; i<BOXDATA(p_iloc)->i_item_count; i++ )
232 if( BOXDATA(p_iloc)->p_items[i].i_item_id != i_item_id )
233 continue;
235 block_t **pp_append = &p_block;
237 /* Shared prefix data, ex: JPEG */
238 if( p_shared_header )
240 *pp_append = block_Alloc( p_shared_header->data.p_binary->i_blob );
241 if( *pp_append )
243 memcpy( (*pp_append)->p_buffer,
244 p_shared_header->data.p_binary->p_blob,
245 p_shared_header->data.p_binary->i_blob );
246 pp_append = &((*pp_append)->p_next);
250 for( uint16_t j=0; j<BOXDATA(p_iloc)->p_items[i].i_extent_count; j++ )
252 uint64_t i_offset = BOXDATA(p_iloc)->p_items[i].i_base_offset +
253 BOXDATA(p_iloc)->p_items[i].p_extents[j].i_extent_offset;
254 uint64_t i_length = BOXDATA(p_iloc)->p_items[i].p_extents[j].i_extent_length;
256 if( BOXDATA(p_iloc)->p_items[i].i_construction_method < 2 )
258 /* Extents are in 1:file, 2:idat */
259 if( BOXDATA(p_iloc)->p_items[i].i_construction_method == 1 )
261 MP4_Box_t *idat = MP4_BoxGet( p_sys->p_root, "meta/idat" );
262 if(!idat)
263 break;
264 i_offset += idat->i_pos + mp4_box_headersize(idat);
267 if( vlc_stream_Seek( p_demux->s, i_offset ) != VLC_SUCCESS )
268 break;
269 *pp_append = vlc_stream_Block( p_demux->s, i_length );
271 /* Extents are 3:iloc reference */
272 else if( BOXDATA(p_iloc)->p_items[i].i_construction_method == 2 )
274 /* FIXME ? That's totally untested and really complicated */
275 uint32_t i_extent_index = BOXDATA(p_iloc)->p_items[i].p_extents[j].i_extent_index;
276 if(i_extent_index == 0)
277 i_extent_index = 1; /* Inferred. Indexes start 1 */
278 const MP4_Box_t *p_iref = MP4_BoxGet( p_sys->p_root, "meta/iref" );
279 if(!p_iref)
280 break;
281 for( const MP4_Box_t *p_refbox = p_iref->p_first;
282 p_refbox; p_refbox = p_refbox->p_next )
284 if( p_refbox->i_type != VLC_FOURCC('i','l','o','c') ||
285 BOXDATA(p_refbox)->i_from_item_id == i_item_id )
286 continue;
288 for( uint16_t k=0; k< BOXDATA(p_refbox)->i_reference_count; k++ )
290 if( --i_extent_index > 0 )
291 continue;
292 if( BOXDATA(p_refbox)->p_references[k].i_to_item_id != i_item_id )
294 *pp_append = ReadItemExtents(p_demux,
295 BOXDATA(p_refbox)->p_references[k].i_to_item_id,
296 NULL);
300 break;
304 while( *pp_append )
305 pp_append = &((*pp_append)->p_next);
307 break;
310 if( p_block )
311 p_block = block_ChainGather( p_block );
313 return p_block;
316 static int SetPictureProperties( demux_t *p_demux, uint32_t i_item_id,
317 es_format_t *fmt, const MP4_Box_t **p_header )
319 struct heif_private_t *p_sys = (void *) p_demux->p_sys;
321 const MP4_Box_t *p_ipma = MP4_BoxGet( p_sys->p_root, "meta/iprp/ipma" );
322 if( !p_ipma )
323 return VLC_EGENERIC;
325 /* Load properties */
326 for( uint32_t i=0; i<BOXDATA(p_ipma)->i_entry_count; i++ )
328 if( BOXDATA(p_ipma)->p_entries[i].i_item_id != i_item_id )
329 continue;
331 for( uint8_t j=0; j<BOXDATA(p_ipma)->p_entries[i].i_association_count; j++ )
333 if( !BOXDATA(p_ipma)->p_entries[i].p_assocs[j].i_property_index )
334 continue;
336 const MP4_Box_t *p_prop = MP4_BoxGet( p_sys->p_root, "meta/iprp/ipco/[%u]",
337 BOXDATA(p_ipma)->p_entries[i].p_assocs[j].i_property_index - 1 );
338 if( !p_prop )
339 continue;
341 switch( p_prop->i_type )
343 case ATOM_hvcC:
344 case ATOM_avcC:
345 case ATOM_av1C:
346 if( !fmt->p_extra &&
347 ((fmt->i_codec == VLC_CODEC_HEVC && p_prop->i_type == ATOM_hvcC) ||
348 (fmt->i_codec == VLC_CODEC_H264 && p_prop->i_type == ATOM_avcC) ||
349 (fmt->i_codec == VLC_CODEC_AV1 && p_prop->i_type == ATOM_av1C)) )
351 fmt->p_extra = malloc( p_prop->data.p_binary->i_blob );
352 if( fmt->p_extra )
354 fmt->i_extra = p_prop->data.p_binary->i_blob;
355 memcpy( fmt->p_extra, p_prop->data.p_binary->p_blob, fmt->i_extra );
358 break;
359 case ATOM_jpeC:
360 if( fmt->i_codec == VLC_CODEC_JPEG )
361 *p_header = p_prop;
362 break;
363 case ATOM_ispe:
364 fmt->video.i_visible_width = p_prop->data.p_ispe->i_width;
365 fmt->video.i_visible_height = p_prop->data.p_ispe->i_height;
366 break;
367 case ATOM_pasp:
368 if( p_prop->data.p_pasp->i_horizontal_spacing &&
369 p_prop->data.p_pasp->i_vertical_spacing )
371 fmt->video.i_sar_num = p_prop->data.p_pasp->i_horizontal_spacing;
372 fmt->video.i_sar_den = p_prop->data.p_pasp->i_vertical_spacing;
374 break;
375 case ATOM_irot:
376 switch( p_prop->data.p_irot->i_ccw_degrees % 360 )
378 default:
379 case 0: fmt->video.orientation = ORIENT_NORMAL ; break;
380 case 90: fmt->video.orientation = ORIENT_ROTATED_90; break;
381 case 180: fmt->video.orientation = ORIENT_ROTATED_180 ; break;
382 case 270: fmt->video.orientation = ORIENT_ROTATED_270 ; break;
384 break;
385 case ATOM_colr:
386 fmt->video.primaries = iso_23001_8_cp_to_vlc_primaries(
387 p_prop->data.p_colr->nclc.i_primary_idx );
388 fmt->video.transfer = iso_23001_8_tc_to_vlc_xfer(
389 p_prop->data.p_colr->nclc.i_transfer_function_idx );
390 fmt->video.space = iso_23001_8_mc_to_vlc_coeffs(
391 p_prop->data.p_colr->nclc.i_matrix_idx );
392 fmt->video.b_color_range_full = p_prop->data.p_colr->nclc.i_full_range;
393 break;
394 case ATOM_clli:
395 fmt->video.lighting.MaxCLL = p_prop->data.p_CoLL->i_maxCLL;
396 fmt->video.lighting.MaxFALL = p_prop->data.p_CoLL->i_maxFALL;
397 break;
398 case ATOM_mdcv:
399 memcpy( fmt->video.mastering.primaries,
400 p_prop->data.p_SmDm->primaries, sizeof(uint16_t) * 6 );
401 memcpy( fmt->video.mastering.white_point,
402 p_prop->data.p_SmDm->white_point, sizeof(uint16_t) * 2 );
403 fmt->video.mastering.max_luminance = p_prop->data.p_SmDm->i_luminanceMax;
404 fmt->video.mastering.min_luminance = p_prop->data.p_SmDm->i_luminanceMin;
405 break;
410 fmt->video.i_frame_rate = 1000;
411 fmt->video.i_frame_rate_base = p_sys->i_image_duration / 1000;
413 return VLC_SUCCESS;
416 static int SetupPicture( demux_t *p_demux, const MP4_Box_t *p_infe,
417 es_format_t *fmt, const MP4_Box_t **p_header )
419 fmt->i_codec = 0;
420 *p_header = NULL;
422 const uint32_t i_item_id = BOXDATA(p_infe)->i_item_id;
423 const char *psz_mime = BOXDATA(p_infe)->psz_content_type;
424 switch( BOXDATA(p_infe)->item_type )
426 case VLC_FOURCC('h','v','c','1'):
427 es_format_Init( fmt, VIDEO_ES, VLC_CODEC_HEVC );
428 break;
429 case VLC_FOURCC('a','v','c','1'):
430 es_format_Init( fmt, VIDEO_ES, VLC_CODEC_H264 );
431 break;
432 case ATOM_av01:
433 es_format_Init( fmt, VIDEO_ES, VLC_CODEC_AV1 );
434 break;
435 case VLC_FOURCC('j','p','e','g'):
436 es_format_Init( fmt, VIDEO_ES, VLC_CODEC_JPEG );
437 break;
438 default:
439 if( psz_mime )
441 if( !strcasecmp( "image/jpeg", psz_mime ) )
442 es_format_Init( fmt, VIDEO_ES, VLC_CODEC_JPEG );
443 else if( !strcasecmp( "image/avif", psz_mime ) )
444 es_format_Init( fmt, VIDEO_ES, VLC_CODEC_AV1 );
446 break;
449 if( fmt->i_codec == 0 )
450 return VLC_EGENERIC;
452 return SetPictureProperties( p_demux, i_item_id, fmt, p_header );
455 static int DemuxHEIF( demux_t *p_demux )
457 struct heif_private_t *p_sys = (void *) p_demux->p_sys;
459 /* Displaying a picture */
460 if( p_sys->i_end_display_time > 0 )
462 bool b_empty;
463 es_out_Control( p_demux->out, ES_OUT_GET_EMPTY, &b_empty );
464 if( !b_empty || vlc_tick_now() <= p_sys->i_end_display_time )
466 vlc_tick_sleep( VLC_TICK_FROM_MS(40) );
467 return VLC_DEMUXER_SUCCESS;
469 p_sys->i_end_display_time = 0;
472 /* Reset prev pic params */
473 p_sys->current.p_shared_header = NULL;
475 /* First or next picture */
476 if( !p_sys->current.p_infe )
478 MP4_Box_t *p_pitm = MP4_BoxGet( p_sys->p_root, "meta/pitm" );
479 if( !p_pitm )
480 return VLC_DEMUXER_EOF;
482 p_sys->current.p_infe = GetAtom( p_sys->p_root, NULL,
483 ATOM_infe, "meta/iinf/infe",
484 MatchInfeID, &BOXDATA(p_pitm)->i_item_id );
486 else
488 p_sys->current.p_infe = GetAtom( p_sys->p_root, p_sys->current.p_infe,
489 ATOM_infe, "meta/iinf/infe",
490 MatchPureImage, p_sys->p_root );
493 if( !p_sys->current.p_infe )
494 return VLC_DEMUXER_EOF;
496 const uint32_t i_current_item_id = p_sys->current.BOXDATA(p_infe)->i_item_id;
497 const MP4_Box_t *p_ipco = MP4_BoxGet( p_sys->p_root, "meta/iprp/ipco" );
498 if( !p_ipco )
499 return VLC_DEMUXER_EOF;
501 es_format_t fmt;
502 es_format_Init(&fmt, UNKNOWN_ES, 0);
504 if( SetupPicture( p_demux, p_sys->current.p_infe,
505 &fmt, &p_sys->current.p_shared_header ) != VLC_SUCCESS )
507 es_format_Clean( &fmt );
508 return VLC_DEMUXER_SUCCESS; /* Unsupported picture, goto next */
511 es_format_Clean( &p_sys->current.fmt );
512 es_format_Copy( &p_sys->current.fmt, &fmt );
513 es_format_Clean( &fmt );
514 if( !p_sys->id )
515 p_sys->id = es_out_Add( p_demux->out, &p_sys->current.fmt );
516 else
517 es_out_Control( p_demux->out, ES_OUT_RESTART_ES, p_sys->id );
519 if( !p_sys->id )
521 p_sys->current.p_infe = NULL; /* Goto next picture */
522 return VLC_DEMUXER_SUCCESS;
525 block_t *p_block = ReadItemExtents( p_demux, i_current_item_id,
526 p_sys->current.p_shared_header );
527 if( !p_block )
528 return VLC_DEMUXER_SUCCESS; /* Goto next picture */
530 if( p_sys->i_pcr == VLC_TICK_INVALID )
532 p_sys->i_pcr = VLC_TICK_0;
533 es_out_SetPCR( p_demux->out, p_sys->i_pcr );
536 p_block->i_dts = p_block->i_pts = p_sys->i_pcr;
537 p_block->i_length = p_sys->i_image_duration;
539 p_block->i_flags |= BLOCK_FLAG_END_OF_SEQUENCE;
541 p_sys->i_end_display_time = vlc_tick_now() + p_block->i_length;
542 p_sys->b_seekpoint_changed = true;
544 p_sys->i_pcr = p_block->i_dts + p_block->i_length;
545 es_out_Send( p_demux->out, p_sys->id, p_block );
546 es_out_SetPCR( p_demux->out, p_sys->i_pcr );
548 return VLC_DEMUXER_SUCCESS;
551 int OpenHEIF( vlc_object_t * p_this )
553 demux_t *p_demux = (demux_t *)p_this;
554 const uint8_t *p_peek;
556 if( vlc_stream_Peek( p_demux->s, &p_peek, 12 ) < 12 )
557 return VLC_EGENERIC;
559 if( VLC_FOURCC( p_peek[4], p_peek[5], p_peek[6], p_peek[7] ) != ATOM_ftyp )
560 return VLC_EGENERIC;
562 switch( VLC_FOURCC( p_peek[8], p_peek[9], p_peek[10], p_peek[11] ) )
564 case BRAND_mif1:
565 case BRAND_heic:
566 case BRAND_heix:
567 case BRAND_jpeg:
568 case BRAND_avci:
569 case BRAND_avif:
570 break;
571 case BRAND_msf1:
572 case BRAND_hevc:
573 case BRAND_hevx:
574 case BRAND_avcs:
575 case BRAND_avis:
576 default:
577 return VLC_EGENERIC;
580 MP4_Box_t *p_root = MP4_BoxGetRoot( p_demux->s );
581 if( !p_root )
582 return VLC_EGENERIC;
584 MP4_BoxDumpStructure( p_demux->s, p_root );
586 struct heif_private_t *p_sys = calloc( 1, sizeof(*p_sys) );
587 p_demux->p_sys = (void *) p_sys;
588 p_sys->p_root = p_root;
589 p_sys->p_title = vlc_input_title_New();
590 if( !p_sys->p_title )
592 free( p_sys );
593 return VLC_ENOMEM;
596 p_sys->i_image_duration = vlc_tick_from_sec(var_InheritFloat( p_demux, "heif-image-duration" ));
597 if( p_sys->i_image_duration <= 0 )
598 p_sys->i_image_duration = VLC_TICK_FROM_SEC(HEIF_DEFAULT_DURATION);
600 MP4_Box_t *p_infe = NULL;
601 while( (p_infe = NextAtom( p_root, ATOM_infe, "meta/iinf/infe", p_infe )) )
603 if( (BOXDATA(p_infe)->i_flags & 0x01) != 0x00 ||
604 !MatchPureImage( p_infe, p_root ) )
605 continue;
606 seekpoint_t *s = vlc_seekpoint_New();
607 if( s )
609 s->i_time_offset = p_sys->p_title->i_seekpoint * p_sys->i_image_duration;
610 if( BOXDATA(p_infe)->psz_item_name )
611 s->psz_name = strdup( BOXDATA(p_infe)->psz_item_name );
612 TAB_APPEND( p_sys->p_title->i_seekpoint, p_sys->p_title->seekpoint, s );
616 es_format_Init( &p_sys->current.fmt, UNKNOWN_ES, 0 );
618 p_demux->pf_demux = DemuxHEIF;
619 p_demux->pf_control = ControlHEIF;
621 return VLC_SUCCESS;
624 void CloseHEIF ( vlc_object_t * p_this )
626 demux_t *p_demux = (demux_t *)p_this;
627 struct heif_private_t *p_sys = (void *) p_demux->p_sys;
628 MP4_BoxFree( p_sys->p_root );
629 if( p_sys->id )
630 es_out_Del( p_demux->out, p_sys->id );
631 es_format_Clean( &p_sys->current.fmt );
632 vlc_input_title_Delete( p_sys->p_title );
633 free( p_sys );