modules/demux/mp4/heif.c

   1 /*****************************************************************************
   2  * heif.c : ISO/IEC 23008-12 HEIF still picture demuxer
   3  *****************************************************************************
   4  * Copyright (C) 2018 Videolabs, VLC authors and VideoLAN
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published by
   8  * the Free Software Foundation; either version 2.1 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  * GNU Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  19  *****************************************************************************/
  20 #ifdef HAVE_CONFIG_H
  21 # include "config.h"
  22 #endif
  23
  24 /*****************************************************************************
  25  * Preamble
  26  *****************************************************************************/
  27 #include <vlc_common.h>
  28 #include <vlc_demux.h>
  29 #include <vlc_input.h>
  30 #include <assert.h>
  31 #include <limits.h>
  32
  33 #include "libmp4.h"
  34 #include "heif.h"
  35 #include "color_config.h"
  36
  37 struct heif_private_t
  38 {
  39     MP4_Box_t *p_root;
  40     es_out_id_t *id;
  41     vlc_tick_t i_pcr;
  42     vlc_tick_t i_end_display_time;
  43     vlc_tick_t i_image_duration;
  44     bool b_seekpoint_changed;
  45     uint32_t i_seekpoint;
  46     input_title_t *p_title;
  47
  48     struct
  49     {
  50         MP4_Box_t *p_infe;
  51         es_format_t fmt;
  52         const MP4_Box_t *p_shared_header;
  53     } current;
  54 };
  55
  56 static MP4_Box_t * NextAtom( MP4_Box_t *p_root,
  57                              vlc_fourcc_t i_type, const char *psz_path,
  58                              MP4_Box_t *p_infe )
  59 {
  60     if( p_infe == NULL )
  61         p_infe = MP4_BoxGet( p_root, psz_path );
  62     else
  63         p_infe = p_infe->p_next;
  64     for( ; p_infe; p_infe = p_infe->p_next )
  65     {
  66         if( p_infe->i_type == i_type )
  67             return p_infe;
  68     }
  69     return NULL;
  70 }
  71
  72 static MP4_Box_t * GetAtom( MP4_Box_t *p_root, MP4_Box_t *p_atom,
  73                             vlc_fourcc_t i_type, const char *psz_path,
  74                             bool(*pf_match)(const MP4_Box_t *, void *),
  75                             void *priv )
  76 {
  77     while( (p_atom = NextAtom( p_root, i_type, psz_path, p_atom )) )
  78     {
  79         if( pf_match( p_atom, priv ) )
  80             return p_atom;
  81     }
  82     return NULL;
  83 }
  84
  85 static bool MatchInfeID( const MP4_Box_t *p_infe, void *priv )
  86 {
  87     return BOXDATA(p_infe)->i_item_id == *((uint32_t *) priv);
  88 }
  89
  90 static bool MatchPureImage( const MP4_Box_t *p_infe, void *priv )
  91 {
  92     MP4_Box_t *p_root = priv;
  93     const MP4_Box_t *p_iref = MP4_BoxGet( p_root, "meta/iref" );
  94     if( !p_iref )
  95         return true;
  96     for( const MP4_Box_t *p_refbox = p_iref->p_first;
  97                           p_refbox; p_refbox = p_refbox->p_next )
  98     {
  99         if( BOXDATA(p_refbox)->i_from_item_id == BOXDATA(p_infe)->i_item_id )
 100             return false;
 101     }
 102     return true;
 103 }
 104
 105 static void SeekToPrevImageEnd( struct heif_private_t *p_sys, int i_picture )
 106 {
 107     int i = 0;
 108     MP4_Box_t *p_infe = NULL;
 109     while( i < i_picture &&
 110           (p_infe = NextAtom( p_sys->p_root, ATOM_infe, "meta/iinf/infe", p_infe )) )
 111     {
 112         if( (BOXDATA(p_infe)->i_flags & 0x01) != 0x00 ||
 113                 !MatchPureImage( p_infe, p_sys->p_root ) )
 114             continue;
 115         i++;
 116     }
 117     p_sys->current.p_infe = p_infe;
 118     p_sys->i_end_display_time = 0;
 119     p_sys->i_pcr = i * p_sys->i_image_duration;
 120 }
 121
 122 static int ControlHEIF( demux_t *p_demux, int i_query, va_list args )
 123 {
 124     struct heif_private_t *p_sys = (void *) p_demux->p_sys;
 125
 126     switch( i_query )
 127     {
 128         case DEMUX_CAN_SEEK:
 129             *va_arg(args, bool *) = true;
 130             return VLC_SUCCESS;
 131         case DEMUX_GET_TITLE_INFO:
 132         {
 133             input_title_t ***ppp_title = va_arg( args, input_title_t *** );
 134             int *pi_int = va_arg( args, int* );
 135             int *pi_title_offset = va_arg( args, int* );
 136             int *pi_seekpoint_offset = va_arg( args, int* );
 137
 138             if( !p_sys->p_title )
 139                 return VLC_EGENERIC;
 140
 141             *pi_int = 1;
 142             *ppp_title = malloc( sizeof( input_title_t*) );
 143             (*ppp_title)[0] = vlc_input_title_Duplicate( p_sys->p_title );
 144             *pi_title_offset = 0;
 145             *pi_seekpoint_offset = 0;
 146             return VLC_SUCCESS;
 147         }
 148         case DEMUX_SET_TITLE:
 149         {
 150             const int i_title = va_arg( args, int );
 151             if( !p_sys->p_title || i_title != 0 )
 152                 return VLC_EGENERIC;
 153             return VLC_SUCCESS;
 154         }
 155         case DEMUX_GET_SEEKPOINT:
 156             *va_arg( args, int * ) = p_sys->i_seekpoint;
 157             return VLC_SUCCESS;
 158         case DEMUX_SET_SEEKPOINT:
 159         {
 160             const int i_seekpoint = va_arg( args, int );
 161             if( !p_sys->p_title )
 162                 return VLC_EGENERIC;
 163             SeekToPrevImageEnd( p_sys, i_seekpoint );
 164             return VLC_SUCCESS;
 165         }
 166         case DEMUX_TEST_AND_CLEAR_FLAGS:
 167         {
 168             unsigned *restrict flags = va_arg( args, unsigned * );
 169
 170             if ((*flags & INPUT_UPDATE_SEEKPOINT) && p_sys->b_seekpoint_changed)
 171             {
 172                 *flags = INPUT_UPDATE_SEEKPOINT;
 173                 p_sys->b_seekpoint_changed = false;
 174             }
 175             else
 176                 *flags = 0;
 177             return VLC_SUCCESS;
 178         }
 179         case DEMUX_GET_LENGTH:
 180             *(va_arg( args, int64_t * )) = p_sys->p_title->i_seekpoint *
 181                                            p_sys->i_image_duration;
 182             return VLC_SUCCESS;
 183         case DEMUX_GET_TIME:
 184             *(va_arg(args, int64_t *)) = p_sys->i_pcr;
 185             return VLC_SUCCESS;
 186         case DEMUX_SET_TIME:
 187         {
 188             SeekToPrevImageEnd( p_sys, va_arg(args, int64_t) /
 189                                 p_sys->i_image_duration );
 190             return VLC_SUCCESS;
 191         }
 192         case DEMUX_GET_POSITION:
 193             if( !p_sys->p_title->i_seekpoint )
 194                 return VLC_EGENERIC;
 195             *(va_arg(args, double *)) = (double) p_sys->i_pcr /
 196                     (p_sys->p_title->i_seekpoint * p_sys->i_image_duration);
 197             return VLC_SUCCESS;
 198         case DEMUX_SET_POSITION:
 199         {
 200             SeekToPrevImageEnd( p_sys,  va_arg(args, double) * p_sys->p_title->i_seekpoint );
 201             return VLC_SUCCESS;
 202         }
 203         case DEMUX_CAN_PAUSE:
 204         case DEMUX_SET_PAUSE_STATE:
 205         case DEMUX_CAN_CONTROL_PACE:
 206         case DEMUX_GET_PTS_DELAY:
 207             return demux_vaControlHelper( p_demux->s, 0, -1, 0, 1, i_query, args );
 208
 209         default:
 210             return VLC_EGENERIC;
 211
 212     }
 213 }
 214
 215 static int DemuxHEIF( demux_t *p_demux )
 216 {
 217     struct heif_private_t *p_sys = (void *) p_demux->p_sys;
 218
 219     /* Displaying a picture */
 220     if( p_sys->i_end_display_time > 0 )
 221     {
 222         bool b_empty;
 223         es_out_Control( p_demux->out, ES_OUT_GET_EMPTY, &b_empty );
 224         if( !b_empty || vlc_tick_now() <= p_sys->i_end_display_time )
 225         {
 226             vlc_tick_sleep( VLC_TICK_FROM_MS(40) );
 227             return VLC_DEMUXER_SUCCESS;
 228         }
 229         p_sys->i_end_display_time = 0;
 230     }
 231
 232     /* Reset prev pic params */
 233     p_sys->current.p_shared_header = NULL;
 234
 235     /* First or next picture */
 236     if( !p_sys->current.p_infe )
 237     {
 238         MP4_Box_t *p_pitm = MP4_BoxGet( p_sys->p_root, "meta/pitm" );
 239         if( !p_pitm )
 240             return VLC_DEMUXER_EOF;
 241
 242         p_sys->current.p_infe = GetAtom( p_sys->p_root, NULL,
 243                                          ATOM_infe, "meta/iinf/infe",
 244                                          MatchInfeID, &BOXDATA(p_pitm)->i_item_id );
 245     }
 246     else
 247     {
 248         p_sys->current.p_infe = GetAtom( p_sys->p_root, p_sys->current.p_infe,
 249                                          ATOM_infe, "meta/iinf/infe",
 250                                          MatchPureImage, p_sys->p_root );
 251     }
 252
 253     if( !p_sys->current.p_infe )
 254         return VLC_DEMUXER_EOF;
 255
 256     const uint32_t i_current_item_id = p_sys->current.BOXDATA(p_infe)->i_item_id;
 257     const MP4_Box_t *p_ipco = MP4_BoxGet( p_sys->p_root, "meta/iprp/ipco" );
 258     const MP4_Box_t *p_ipma = MP4_BoxGet( p_sys->p_root, "meta/iprp/ipma" );
 259     if( !p_ipma || !p_ipco )
 260         return VLC_DEMUXER_EOF;
 261
 262     es_format_t fmt;
 263     const char *psz_mime = p_sys->current.BOXDATA(p_infe)->psz_content_type;
 264     switch( p_sys->current.BOXDATA(p_infe)->item_type )
 265     {
 266         case VLC_FOURCC('h','v','c','1'):
 267             es_format_Init( &fmt, VIDEO_ES, VLC_CODEC_HEVC );
 268             break;
 269         case VLC_FOURCC('a','v','c','1'):
 270             es_format_Init( &fmt, VIDEO_ES, VLC_CODEC_H264 );
 271             break;
 272         case VLC_FOURCC('j','p','e','g'):
 273             es_format_Init( &fmt, VIDEO_ES, VLC_CODEC_JPEG );
 274             break;
 275         default:
 276             if( psz_mime && !strcasecmp( "image/jpeg", psz_mime ) )
 277             {
 278                 es_format_Init( &fmt, VIDEO_ES, VLC_CODEC_JPEG );
 279                 break;
 280             }
 281             return VLC_DEMUXER_SUCCESS; /* Unsupported picture, goto next */
 282     }
 283
 284     /* Load properties */
 285     for( uint32_t i=0; i<BOXDATA(p_ipma)->i_entry_count; i++ )
 286     {
 287         if( BOXDATA(p_ipma)->p_entries[i].i_item_id != i_current_item_id )
 288             continue;
 289         for( uint8_t j=0; j<BOXDATA(p_ipma)->p_entries[i].i_association_count; j++ )
 290         {
 291             if( !BOXDATA(p_ipma)->p_entries[i].p_assocs[j].i_property_index )
 292                 continue;
 293
 294             const MP4_Box_t *p_prop = MP4_BoxGet( p_ipco, "./[%u]",
 295                 BOXDATA(p_ipma)->p_entries[i].p_assocs[j].i_property_index - 1 );
 296             if( !p_prop )
 297                 continue;
 298
 299             switch( p_prop->i_type )
 300             {
 301                 case ATOM_hvcC:
 302                 case ATOM_avcC:
 303                     if( !fmt.p_extra &&
 304                        ((fmt.i_codec == VLC_CODEC_HEVC && p_prop->i_type == ATOM_hvcC) ||
 305                         (fmt.i_codec == VLC_CODEC_H264 && p_prop->i_type == ATOM_avcC)) )
 306                     {
 307                         fmt.p_extra = malloc( p_prop->data.p_binary->i_blob );
 308                         if( fmt.p_extra )
 309                         {
 310                             fmt.i_extra = p_prop->data.p_binary->i_blob;
 311                             memcpy( fmt.p_extra, p_prop->data.p_binary->p_blob, fmt.i_extra );
 312                         }
 313                     }
 314                     break;
 315                 case ATOM_jpeC:
 316                     if( fmt.i_codec == VLC_CODEC_JPEG )
 317                         p_sys->current.p_shared_header = p_prop;
 318                     break;
 319                 case ATOM_ispe:
 320                     fmt.video.i_visible_width = p_prop->data.p_ispe->i_width;
 321                     fmt.video.i_visible_height = p_prop->data.p_ispe->i_height;
 322                     break;
 323                 case ATOM_pasp:
 324                     if( p_prop->data.p_pasp->i_horizontal_spacing &&
 325                         p_prop->data.p_pasp->i_vertical_spacing )
 326                     {
 327                         fmt.video.i_sar_num = p_prop->data.p_pasp->i_horizontal_spacing;
 328                         fmt.video.i_sar_den = p_prop->data.p_pasp->i_vertical_spacing;
 329                     }
 330                     break;
 331                 case ATOM_irot:
 332                     switch( p_prop->data.p_irot->i_ccw_degrees % 360 )
 333                     {
 334                         default:
 335                         case 0:   fmt.video.orientation = ORIENT_NORMAL ; break;
 336                         case 90:  fmt.video.orientation = ORIENT_ROTATED_90; break;
 337                         case 180: fmt.video.orientation = ORIENT_ROTATED_180 ; break;
 338                         case 270: fmt.video.orientation = ORIENT_ROTATED_270 ; break;
 339                     }
 340                     break;
 341                 case ATOM_colr:
 342                     fmt.video.primaries = iso_23001_8_cp_to_vlc_primaries(
 343                                             p_prop->data.p_colr->nclc.i_primary_idx );
 344                     fmt.video.transfer = iso_23001_8_tc_to_vlc_xfer(
 345                                             p_prop->data.p_colr->nclc.i_transfer_function_idx );
 346                     fmt.video.space = iso_23001_8_mc_to_vlc_coeffs(
 347                                         p_prop->data.p_colr->nclc.i_matrix_idx );
 348                     fmt.video.b_color_range_full = p_prop->data.p_colr->nclc.i_full_range;
 349                     break;
 350             }
 351         }
 352     }
 353
 354     fmt.video.i_frame_rate      = 1000;
 355     fmt.video.i_frame_rate_base = p_sys->i_image_duration / 1000;
 356
 357     es_format_Clean( &p_sys->current.fmt );
 358     es_format_Copy( &p_sys->current.fmt, &fmt );
 359     es_format_Clean( &fmt );
 360     if( !p_sys->id )
 361         p_sys->id = es_out_Add( p_demux->out, &p_sys->current.fmt );
 362     else
 363         es_out_Control( p_demux->out, ES_OUT_RESTART_ES, p_sys->id );
 364
 365     if( !p_sys->id )
 366     {
 367         p_sys->current.p_infe = NULL; /* Goto next picture */
 368         return VLC_DEMUXER_SUCCESS;
 369     }
 370
 371     MP4_Box_t *p_iloc = MP4_BoxGet( p_sys->p_root, "meta/iloc" );
 372     if( !p_iloc )
 373         return VLC_DEMUXER_EOF;
 374
 375     block_t *p_block = NULL;
 376     for( uint32_t i=0; i<BOXDATA(p_iloc)->i_item_count; i++ )
 377     {
 378         if( BOXDATA(p_iloc)->p_items[i].i_item_id != i_current_item_id )
 379             continue;
 380
 381         block_t **pp_append = &p_block;
 382
 383         /* Shared prefix data, ex: JPEG */
 384         if( p_sys->current.p_shared_header )
 385         {
 386             *pp_append = block_Alloc( p_sys->current.p_shared_header->data.p_binary->i_blob );
 387             if( *pp_append )
 388             {
 389                 memcpy( (*pp_append)->p_buffer,
 390                         p_sys->current.p_shared_header->data.p_binary->p_blob,
 391                         p_sys->current.p_shared_header->data.p_binary->i_blob );
 392                 pp_append = &((*pp_append)->p_next);
 393             }
 394         }
 395
 396         for( uint16_t j=0; j<BOXDATA(p_iloc)->p_items[i].i_extent_count; j++ )
 397         {
 398             uint64_t i_offset = BOXDATA(p_iloc)->p_items[i].i_base_offset +
 399                                 BOXDATA(p_iloc)->p_items[i].p_extents[j].i_extent_offset;
 400             uint64_t i_length = BOXDATA(p_iloc)->p_items[i].p_extents[j].i_extent_length;
 401             if( vlc_stream_Seek( p_demux->s, i_offset ) != VLC_SUCCESS )
 402                 break;
 403             *pp_append = vlc_stream_Block( p_demux->s, i_length );
 404             if( *pp_append )
 405                 pp_append = &((*pp_append)->p_next);
 406         }
 407         break;
 408     }
 409
 410     if( p_block )
 411         p_block = block_ChainGather( p_block );
 412     if( !p_block )
 413         return VLC_DEMUXER_SUCCESS; /* Goto next picture */
 414
 415     if( p_sys->i_pcr == VLC_TICK_INVALID )
 416     {
 417         p_sys->i_pcr = VLC_TICK_0;
 418         es_out_SetPCR( p_demux->out, p_sys->i_pcr );
 419     }
 420
 421     p_block->i_dts = p_block->i_pts = p_sys->i_pcr;
 422     p_block->i_length = p_sys->i_image_duration;
 423
 424     p_block->i_flags |= BLOCK_FLAG_END_OF_SEQUENCE;
 425
 426     p_sys->i_end_display_time = vlc_tick_now() + p_block->i_length;
 427     p_sys->b_seekpoint_changed = true;
 428
 429     p_sys->i_pcr = p_block->i_dts + p_block->i_length;
 430     es_out_Send( p_demux->out, p_sys->id, p_block );
 431     es_out_SetPCR( p_demux->out, p_sys->i_pcr );
 432
 433     return VLC_DEMUXER_SUCCESS;
 434 }
 435
 436 int OpenHEIF( vlc_object_t * p_this )
 437 {
 438     demux_t  *p_demux = (demux_t *)p_this;
 439     const uint8_t *p_peek;
 440
 441     if( vlc_stream_Peek( p_demux->s, &p_peek, 12 ) < 12 )
 442         return VLC_EGENERIC;
 443
 444     if( VLC_FOURCC( p_peek[4], p_peek[5], p_peek[6], p_peek[7] ) != ATOM_ftyp )
 445         return VLC_EGENERIC;
 446
 447     switch( VLC_FOURCC( p_peek[8], p_peek[9], p_peek[10], p_peek[11] ) )
 448     {
 449         case MAJOR_mif1:
 450         case MAJOR_heic:
 451         case MAJOR_heix:
 452         case MAJOR_jpeg:
 453         case MAJOR_avci:
 454             break;
 455         case MAJOR_msf1:
 456         case MAJOR_hevc:
 457         case MAJOR_hevx:
 458         case MAJOR_avcs:
 459         default:
 460             return VLC_EGENERIC;
 461     }
 462
 463     MP4_Box_t *p_root = MP4_BoxGetRoot( p_demux->s );
 464     if( !p_root )
 465         return VLC_EGENERIC;
 466
 467     MP4_BoxDumpStructure( p_demux->s, p_root );
 468
 469     struct heif_private_t *p_sys = calloc( 1, sizeof(*p_sys) );
 470     p_demux->p_sys = (void *) p_sys;
 471     p_sys->p_root = p_root;
 472     p_sys->p_title = vlc_input_title_New();
 473     if( !p_sys->p_title )
 474     {
 475         free( p_sys );
 476         return VLC_ENOMEM;
 477     }
 478
 479     p_sys->i_image_duration = var_InheritFloat( p_demux, "heif-image-duration" );
 480     if( p_sys->i_image_duration <= 0 )
 481         p_sys->i_image_duration = HEIF_DEFAULT_DURATION;
 482     p_sys->i_image_duration *= CLOCK_FREQ;
 483
 484     MP4_Box_t *p_infe = NULL;
 485     while( (p_infe = NextAtom( p_root, ATOM_infe, "meta/iinf/infe", p_infe )) )
 486     {
 487         if( (BOXDATA(p_infe)->i_flags & 0x01) != 0x00 ||
 488                 !MatchPureImage( p_infe, p_root ) )
 489             continue;
 490         seekpoint_t *s = vlc_seekpoint_New();
 491         if( s )
 492         {
 493             s->i_time_offset = p_sys->p_title->i_seekpoint * p_sys->i_image_duration;
 494             if( BOXDATA(p_infe)->psz_item_name )
 495                 s->psz_name = strdup( BOXDATA(p_infe)->psz_item_name );
 496             TAB_APPEND( p_sys->p_title->i_seekpoint, p_sys->p_title->seekpoint, s );
 497         }
 498     }
 499
 500     es_format_Init( &p_sys->current.fmt, UNKNOWN_ES, 0 );
 501
 502     p_demux->pf_demux = DemuxHEIF;
 503     p_demux->pf_control = ControlHEIF;
 504
 505     return VLC_SUCCESS;
 506 }
 507
 508 void CloseHEIF ( vlc_object_t * p_this )
 509 {
 510     demux_t *p_demux = (demux_t *)p_this;
 511     struct heif_private_t *p_sys = (void *) p_demux->p_sys;
 512     MP4_BoxFree( p_sys->p_root );
 513     if( p_sys->id )
 514         es_out_Del( p_demux->out, p_sys->id );
 515     es_format_Clean( &p_sys->current.fmt );
 516     vlc_input_title_Delete( p_sys->p_title );
 517     free( p_sys );
 518 }