modules/packetizer/av1.c

   1 /*****************************************************************************
   2  * av1.c: AV1 video packetizer
   3  *****************************************************************************
   4  * Copyright (C) 2018 VideoLabs, VLC authors and VideoLAN
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU Lesser General Public License as published by
   8  * the Free Software Foundation; either version 2.1 of the License, or
   9  * (at your option) any later version.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  14  * GNU Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public License
  17  * along with this program; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  19  *****************************************************************************/
  20
  21 /*****************************************************************************
  22  * Preamble
  23  *****************************************************************************/
  24
  25 #ifdef HAVE_CONFIG_H
  26 # include "config.h"
  27 #endif
  28
  29 #include <vlc_common.h>
  30 #include <vlc_plugin.h>
  31 #include <vlc_codec.h>
  32 #include <vlc_block.h>
  33 #include <vlc_bits.h>
  34
  35 #include <vlc_block_helper.h>
  36
  37 #include "av1.h"
  38 #include "av1_obu.h"
  39
  40 //#define DEBUG_AV1_PACKETIZER
  41
  42 /****************************************************************************
  43  * Local prototypes
  44  ****************************************************************************/
  45 typedef struct
  46 {
  47     struct
  48     {
  49         block_t *p_chain;
  50         block_t **pp_chain_last;
  51     } obus;
  52
  53     block_t *p_sequence_header_block;
  54     av1_OBU_sequence_header_t *p_sequence_header;
  55     struct
  56     {
  57         bool b_has_visible_frame;
  58         struct
  59         {
  60             block_t *p_chain;
  61             block_t **pp_chain_last;
  62         } pre, frame, post;
  63         vlc_tick_t dts;
  64         vlc_tick_t pts;
  65     } tu;
  66     uint32_t i_seen;
  67     int i_next_block_flags;
  68
  69 } av1_sys_t;
  70
  71 #define BLOCK_FLAG_DROP (1 << BLOCK_FLAG_PRIVATE_SHIFT)
  72
  73 /****************************************************************************
  74  * Helpers
  75  ****************************************************************************/
  76 static inline void InitQueue(block_t **pp_head, block_t ***ppp_tail)
  77 {
  78     *pp_head = NULL;
  79     *ppp_tail = pp_head;
  80 }
  81
  82 static bool block_Differs(const block_t *a, const block_t *b)
  83 {
  84     return (a->i_buffer != b->i_buffer ||
  85             memcmp(a->p_buffer, b->p_buffer, a->i_buffer));
  86 }
  87
  88 #define INITQ(name) InitQueue(&p_sys->name.p_chain, &p_sys->name.pp_chain_last)
  89 #define PUSHQ(name,b) \
  90 {\
  91     block_ChainLastAppend(&p_sys->name.pp_chain_last, b);\
  92     if(p_sys->tu.dts == VLC_TICK_INVALID)\
  93         p_sys->tu.dts = b->i_dts; p_sys->tu.pts = b->i_pts;\
  94 }
  95
  96 static void UpdateDecoderFormat(decoder_t *p_dec)
  97 {
  98     av1_sys_t *p_sys = p_dec->p_sys;
  99     if(!p_sys->p_sequence_header)
 100         return;
 101
 102     if(p_dec->fmt_in.i_profile < AV1_PROFILE_MAIN)
 103     {
 104         int val[3];
 105         AV1_get_profile_level(p_sys->p_sequence_header, &val[0], &val[1], &val[2]);
 106         if(p_dec->fmt_out.i_profile != val[0] || p_dec->fmt_out.i_level != val[1])
 107         {
 108             p_dec->fmt_out.i_profile = val[0];
 109             p_dec->fmt_out.i_level = val[1];
 110         }
 111     }
 112
 113     unsigned wnum, hden;
 114     AV1_get_frame_max_dimensions(p_sys->p_sequence_header, &wnum, &hden);
 115     if((!p_dec->fmt_in.video.i_visible_height ||
 116         !p_dec->fmt_in.video.i_visible_width) &&
 117        (p_dec->fmt_out.video.i_visible_width != wnum ||
 118         p_dec->fmt_out.video.i_visible_width != hden))
 119     {
 120         p_dec->fmt_out.video.i_width =
 121         p_dec->fmt_out.video.i_visible_width = wnum;
 122         p_dec->fmt_out.video.i_height =
 123         p_dec->fmt_out.video.i_visible_height = hden;
 124     }
 125
 126     if((!p_dec->fmt_in.video.i_frame_rate ||
 127         !p_dec->fmt_in.video.i_frame_rate_base) &&
 128         AV1_get_frame_rate(p_sys->p_sequence_header, &wnum, &hden) &&
 129         (p_dec->fmt_out.video.i_frame_rate != wnum ||
 130          p_dec->fmt_out.video.i_frame_rate_base != hden))
 131     {
 132         p_dec->fmt_out.video.i_frame_rate = wnum;
 133         p_dec->fmt_out.video.i_frame_rate_base = hden;
 134     }
 135
 136     video_color_primaries_t prim;
 137     video_color_space_t space;
 138     video_transfer_func_t xfer;
 139     video_color_range_t full;
 140     if(p_dec->fmt_in.video.primaries == COLOR_PRIMARIES_UNDEF &&
 141        AV1_get_colorimetry(p_sys->p_sequence_header, &prim, &xfer, &space, &full) &&
 142        prim != COLOR_PRIMARIES_UNDEF &&
 143        (p_dec->fmt_out.video.primaries != prim ||
 144         p_dec->fmt_out.video.transfer != xfer ||
 145         p_dec->fmt_out.video.space != space))
 146     {
 147         p_dec->fmt_out.video.primaries = prim;
 148         p_dec->fmt_out.video.transfer = xfer;
 149         p_dec->fmt_out.video.space = space;
 150         p_dec->fmt_out.video.color_range = full;
 151     }
 152
 153     if(!p_dec->fmt_in.i_extra && !p_dec->fmt_out.i_extra)
 154     {
 155         p_dec->fmt_out.i_extra =
 156                 AV1_create_DecoderConfigurationRecord((uint8_t **)&p_dec->fmt_out.p_extra,
 157                                                       p_sys->p_sequence_header,
 158                                                       1,
 159                                                       (const uint8_t **)&p_sys->p_sequence_header_block->p_buffer,
 160                                                       &p_sys->p_sequence_header_block->i_buffer);
 161     }
 162 }
 163
 164 static block_t * OutputQueues(decoder_t *p_dec, bool b_valid)
 165 {
 166     av1_sys_t *p_sys = p_dec->p_sys;
 167     block_t *p_output = NULL;
 168     block_t **pp_output_last = &p_output;
 169     uint32_t i_flags = 0; /* Because block_ChainGather does not merge flags or times */
 170
 171     if(p_sys->tu.pre.p_chain)
 172     {
 173         block_ChainLastAppend(&pp_output_last, p_sys->tu.pre.p_chain);
 174         INITQ(tu.pre);
 175     }
 176
 177     if(p_sys->tu.frame.p_chain)
 178     {
 179         i_flags |= p_sys->tu.frame.p_chain->i_flags;
 180         block_ChainLastAppend(&pp_output_last, p_sys->tu.frame.p_chain);
 181         INITQ(tu.frame);
 182     }
 183
 184     if(p_sys->tu.post.p_chain)
 185     {
 186         block_ChainLastAppend(&pp_output_last, p_sys->tu.post.p_chain);
 187         INITQ(tu.post);
 188     }
 189
 190     if(p_output)
 191     {
 192         p_output->i_dts = p_sys->tu.dts;
 193         p_output->i_pts = p_sys->tu.pts;
 194         p_output->i_flags |= i_flags;
 195         if(!b_valid)
 196             p_output->i_flags |= BLOCK_FLAG_DROP;
 197         else
 198         {
 199             p_output->i_flags |= p_sys->i_next_block_flags;
 200             p_sys->i_next_block_flags = 0;
 201         }
 202     }
 203
 204     p_sys->tu.b_has_visible_frame = false;
 205     p_sys->tu.dts = VLC_TICK_INVALID;
 206     p_sys->tu.pts = VLC_TICK_INVALID;
 207     p_sys->i_seen = 0;
 208
 209     return p_output;
 210 }
 211
 212 /****************************************************************************
 213  * Packetizer Helpers
 214  ****************************************************************************/
 215 static block_t *GatherAndValidateChain(decoder_t *p_dec, block_t *p_outputchain)
 216 {
 217     block_t *p_output = NULL;
 218     av1_sys_t *p_sys = p_dec->p_sys;
 219     VLC_UNUSED(p_sys);
 220
 221     if(p_outputchain)
 222     {
 223 #ifdef DEBUG_AV1_PACKETIZER
 224         msg_Dbg(p_dec, "TU output %ld", p_outputchain->i_dts);
 225         for(block_t *p = p_outputchain; p; p=p->p_next)
 226         {
 227             enum av1_obu_type_e OBUtype = AV1_OBUGetType(p->p_buffer);
 228             if(OBUtype == AV1_OBU_FRAME || OBUtype == AV1_OBU_FRAME_HEADER)
 229             {
 230                 av1_OBU_frame_header_t *p_fh = NULL;
 231                 if(AV1_OBUIsBaseLayer(p->p_buffer, p->i_buffer) && p_sys->p_sequence_header)
 232                 {
 233                     p_fh = AV1_OBU_parse_frame_header(p->p_buffer, p->i_buffer,
 234                                                       p_sys->p_sequence_header);
 235                     if(p_fh)
 236                     {
 237                         msg_Dbg(p_dec,"OBU TYPE %d sz %ld dts %ld type %d %d",
 238                                 OBUtype, p->i_buffer, p->i_dts,
 239                                 AV1_get_frame_type(p_fh),
 240                                 AV1_get_frame_visibility(p_fh));
 241                     }
 242                     AV1_release_frame_header(p_fh);
 243                 }
 244             }
 245             else msg_Dbg(p_dec, "OBU TYPE %d sz %ld dts %ld", OBUtype, p->i_buffer, p->i_dts);
 246         }
 247 #endif
 248         if(p_outputchain->i_flags & BLOCK_FLAG_DROP)
 249             p_output = p_outputchain; /* Avoid useless gather */
 250         else
 251             p_output = block_ChainGather(p_outputchain);
 252     }
 253
 254     if(p_output && (p_output->i_flags & BLOCK_FLAG_DROP))
 255     {
 256         block_ChainRelease(p_output); /* Chain! see above */
 257         p_output = NULL;
 258     }
 259
 260     return p_output;
 261 }
 262
 263 static block_t *ParseOBUBlock(decoder_t *p_dec, block_t *p_obu)
 264 {
 265     av1_sys_t *p_sys = p_dec->p_sys;
 266
 267     block_t * p_output = NULL;
 268     enum av1_obu_type_e OBUtype = AV1_OBUGetType(p_obu->p_buffer);
 269     const bool b_base_layer = AV1_OBUIsBaseLayer(p_obu->p_buffer, p_obu->i_buffer);
 270
 271     switch(OBUtype)
 272     {
 273         case AV1_OBU_SEQUENCE_HEADER:
 274         {
 275             if(p_sys->tu.frame.p_chain || p_sys->tu.post.p_chain)
 276                 p_output = OutputQueues(p_dec, p_sys->p_sequence_header != NULL);
 277
 278             if(b_base_layer)
 279             {
 280                 /* Save a copy for Extradata */
 281                 if(!p_sys->p_sequence_header_block ||
 282                    block_Differs(p_sys->p_sequence_header_block, p_obu))
 283                 {
 284                     if(p_sys->p_sequence_header_block)
 285                         block_Release(p_sys->p_sequence_header_block);
 286                     p_sys->p_sequence_header_block = block_Duplicate(p_obu);
 287                 }
 288
 289                 if(p_sys->p_sequence_header)
 290                     AV1_release_sequence_header(p_sys->p_sequence_header);
 291                 p_sys->p_sequence_header = AV1_OBU_parse_sequence_header(p_obu->p_buffer, p_obu->i_buffer);
 292             }
 293             PUSHQ(tu.pre, p_obu);
 294         } break;
 295
 296         case AV1_OBU_TEMPORAL_DELIMITER:
 297         {
 298             p_output = OutputQueues(p_dec, p_sys->p_sequence_header_block != NULL);
 299             PUSHQ(tu.pre, p_obu);
 300         } break;
 301
 302         case AV1_OBU_FRAME:
 303         case AV1_OBU_FRAME_HEADER:
 304         {
 305             if(b_base_layer)
 306             {
 307                 av1_OBU_frame_header_t *p_fh = NULL;
 308                 if(p_sys->p_sequence_header)
 309                 {
 310                     p_fh = AV1_OBU_parse_frame_header(p_obu->p_buffer, p_obu->i_buffer,
 311                                                       p_sys->p_sequence_header);
 312                     if(p_fh)
 313                     {
 314                         if((p_sys->i_seen & AV1_OBU_TEMPORAL_DELIMITER) && p_sys->tu.b_has_visible_frame)
 315                             p_output = OutputQueues(p_dec, p_sys->p_sequence_header != NULL);
 316
 317                         switch(AV1_get_frame_type(p_fh))
 318                         {
 319                             case AV1_FRAME_TYPE_KEY:
 320                             case AV1_FRAME_TYPE_INTRA_ONLY:
 321                                 p_obu->i_flags |= BLOCK_FLAG_TYPE_I;
 322                                 break;
 323                             case AV1_FRAME_TYPE_INTER:
 324                                 p_obu->i_flags |= BLOCK_FLAG_TYPE_P;
 325                                 break;
 326                             default:
 327                                 break;
 328                         }
 329
 330                         p_sys->tu.b_has_visible_frame |= AV1_get_frame_visibility(p_fh);
 331                         AV1_release_frame_header(p_fh);
 332                     }
 333                     else msg_Warn(p_dec, "could not parse frame header");
 334                 }
 335             }
 336
 337             if(!p_output && p_sys->tu.post.p_chain)
 338                 p_output = OutputQueues(p_dec, p_sys->p_sequence_header != NULL);
 339
 340             PUSHQ(tu.frame, p_obu);
 341         } break;
 342
 343         case AV1_OBU_METADATA:
 344         {
 345             if(p_sys->tu.frame.p_chain || p_sys->tu.post.p_chain)
 346                 p_output = OutputQueues(p_dec, p_sys->p_sequence_header != NULL);
 347             PUSHQ(tu.pre, p_obu);
 348         } break;
 349
 350         case AV1_OBU_TILE_GROUP:
 351         case AV1_OBU_TILE_LIST:
 352             if(p_sys->tu.post.p_chain)
 353                 p_output = OutputQueues(p_dec, p_sys->p_sequence_header != NULL);
 354             PUSHQ(tu.frame, p_obu);
 355             break;
 356
 357         case AV1_OBU_REDUNDANT_FRAME_HEADER:
 358         case AV1_OBU_PADDING:
 359         default:
 360             block_Release(p_obu);
 361             break;
 362     }
 363
 364     if(b_base_layer)
 365         p_sys->i_seen |= 1 << OBUtype;
 366
 367     return p_output;
 368 }
 369
 370 /****************************************************************************
 371  * Flush
 372  ****************************************************************************/
 373 static void PacketizeFlush(decoder_t *p_dec)
 374 {
 375     av1_sys_t *p_sys = p_dec->p_sys;
 376
 377     block_ChainRelease(OutputQueues(p_dec, false));
 378
 379     if(p_sys->p_sequence_header)
 380     {
 381         AV1_release_sequence_header(p_sys->p_sequence_header);
 382         p_sys->p_sequence_header = NULL;
 383     }
 384     if(p_sys->p_sequence_header_block)
 385     {
 386         block_Release(p_sys->p_sequence_header_block);
 387         p_sys->p_sequence_header_block = NULL;
 388     }
 389
 390     block_ChainRelease(p_sys->obus.p_chain);
 391     INITQ(obus);
 392
 393     p_sys->tu.dts = VLC_TICK_INVALID;
 394     p_sys->tu.pts = VLC_TICK_INVALID;
 395     p_sys->tu.b_has_visible_frame = false;
 396     p_sys->i_seen = 0;
 397     p_sys->i_next_block_flags = BLOCK_FLAG_DISCONTINUITY;
 398 }
 399
 400 /****************************************************************************
 401  * Packetize
 402  ****************************************************************************/
 403 static block_t *PacketizeOBU(decoder_t *p_dec, block_t **pp_block)
 404 {
 405     av1_sys_t *p_sys = p_dec->p_sys;
 406
 407     block_t *p_block = pp_block ? *pp_block : NULL;
 408     if(p_block)
 409     {
 410         if( p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY | BLOCK_FLAG_CORRUPTED) )
 411         {
 412             /* First always drain complete blocks before discontinuity */
 413             block_t *p_drain = PacketizeOBU( p_dec, NULL );
 414             if(p_drain)
 415                 return p_drain;
 416
 417             PacketizeFlush( p_dec );
 418
 419             if( p_block->i_flags & BLOCK_FLAG_CORRUPTED )
 420             {
 421                 block_Release( p_block );
 422                 return NULL;
 423             }
 424         }
 425
 426         if(!AV1_OBUIsValid(p_block->p_buffer, p_block->i_buffer))
 427         {
 428             msg_Warn(p_dec,"fed with invalid OBU");
 429             block_Release(p_block);
 430             return NULL;
 431         }
 432         *pp_block = NULL;
 433         block_ChainLastAppend(&p_sys->obus.pp_chain_last, p_block);
 434     }
 435
 436     block_t *p_output = NULL;
 437     while(p_sys->obus.p_chain)
 438     {
 439         block_t *p_frag = p_sys->obus.p_chain;
 440
 441         AV1_OBU_iterator_ctx_t it;
 442         AV1_OBU_iterator_init(&it, p_frag->p_buffer, p_frag->i_buffer);
 443         const uint8_t *p_obu; size_t i_obu;
 444
 445         if(!AV1_OBU_iterate_next(&it, &p_obu, &i_obu))
 446         {
 447             msg_Warn(p_dec,"Invalid OBU header in sequence, discarding");
 448             /* frag is not OBU, drop */
 449             p_sys->obus.p_chain = p_frag->p_next;
 450             if(p_frag->p_next == NULL)
 451                 p_sys->obus.pp_chain_last = &p_sys->obus.p_chain;
 452             else
 453                 p_frag->p_next = NULL;
 454             block_Release(p_frag);
 455             continue;
 456         }
 457
 458         block_t *p_obublock;
 459         if(i_obu == p_frag->i_buffer)
 460         {
 461             p_sys->obus.p_chain = p_frag->p_next;
 462             if(p_frag->p_next == NULL)
 463                 p_sys->obus.pp_chain_last = &p_sys->obus.p_chain;
 464             else
 465                 p_frag->p_next = NULL;
 466             p_obublock = p_frag;
 467         }
 468         else
 469         {
 470             p_obublock = block_Alloc(i_obu);
 471             memcpy(p_obublock->p_buffer, p_frag->p_buffer, i_obu);
 472             p_frag->i_buffer -= i_obu;
 473             p_frag->p_buffer += i_obu;
 474             p_obublock->i_dts = p_frag->i_dts;
 475             p_obublock->i_pts = p_frag->i_pts;
 476             p_obublock->i_flags = p_frag->i_flags;
 477             p_frag->i_flags = 0;
 478             p_frag->i_dts = VLC_TICK_INVALID;
 479             p_frag->i_pts = VLC_TICK_INVALID;
 480         }
 481
 482         p_output = ParseOBUBlock(p_dec, p_obublock);
 483         if(p_output)
 484             break;
 485     }
 486
 487
 488     if(!p_output && pp_block == NULL)
 489         p_output = OutputQueues(p_dec, p_sys->p_sequence_header_block != NULL);
 490
 491     if(p_output)
 492     {
 493         p_output = GatherAndValidateChain(p_dec, p_output);
 494         UpdateDecoderFormat(p_dec);
 495     }
 496
 497     return p_output;
 498 }
 499
 500 /*****************************************************************************
 501  * Close
 502  *****************************************************************************/
 503 static void Close(vlc_object_t *p_this)
 504 {
 505     decoder_t *p_dec = (decoder_t*)p_this;
 506     av1_sys_t *p_sys = p_dec->p_sys;
 507
 508     PacketizeFlush(p_dec);
 509
 510     free(p_sys);
 511 }
 512
 513 /*****************************************************************************
 514  * Open
 515  *****************************************************************************/
 516 static int Open(vlc_object_t *p_this)
 517 {
 518     decoder_t *p_dec = (decoder_t*)p_this;
 519     av1_sys_t *p_sys;
 520
 521     if (p_dec->fmt_in.i_codec != VLC_CODEC_AV1)
 522         return VLC_EGENERIC;
 523
 524     p_dec->p_sys = p_sys = calloc(1, sizeof(av1_sys_t));
 525     if (!p_dec->p_sys)
 526         return VLC_ENOMEM;
 527
 528     INITQ(obus);
 529     p_sys->p_sequence_header_block = NULL;
 530     p_sys->p_sequence_header = NULL;
 531     p_sys->tu.b_has_visible_frame = false;
 532     p_sys->tu.dts = VLC_TICK_INVALID;
 533     p_sys->tu.pts = VLC_TICK_INVALID;
 534     p_sys->i_seen = 0;
 535     p_sys->i_next_block_flags = 0;
 536     INITQ(tu.pre);
 537     INITQ(tu.frame);
 538     INITQ(tu.post);
 539
 540     /* Copy properties */
 541     es_format_Copy(&p_dec->fmt_out, &p_dec->fmt_in);
 542     p_dec->fmt_out.b_packetized = true;
 543
 544     p_dec->pf_packetize = PacketizeOBU;
 545     p_dec->pf_flush = PacketizeFlush;
 546
 547     return VLC_SUCCESS;
 548 }
 549
 550 /*****************************************************************************
 551  * Module descriptor
 552  *****************************************************************************/
 553
 554 vlc_module_begin ()
 555     set_category(CAT_SOUT)
 556     set_subcategory(SUBCAT_SOUT_PACKETIZER)
 557     set_description(N_("AV1 video packetizer"))
 558     set_capability("packetizer", 50)
 559     set_callbacks(Open, Close)
 560 vlc_module_end ()