modules/codec/aom.c

   1 /*****************************************************************************
   2  * aom.c: libaom decoder (AV1) module
   3  *****************************************************************************
   4  * Copyright (C) 2016 VLC authors and VideoLAN
   5  *
   6  * Authors: Tristan Matthews <tmatth@videolan.org>
   7  * Based on vpx.c by: Rafaël Carré <funman@videolan.org>
   8  *
   9  * This program is free software; you can redistribute it and/or modify it
  10  * under the terms of the GNU Lesser General Public License as published by
  11  * the Free Software Foundation; either version 2.1 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17  * GNU Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public License
  20  * along with this program; if not, write to the Free Software Foundation,
  21  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * Preamble
  26  *****************************************************************************/
  27 #ifdef HAVE_CONFIG_H
  28 # include "config.h"
  29 #endif
  30
  31 #include <vlc_common.h>
  32 #include <vlc_plugin.h>
  33 #include <vlc_codec.h>
  34
  35 #include <aom/aom_decoder.h>
  36 #include <aom/aomdx.h>
  37
  38 #ifdef ENABLE_SOUT
  39 # include <aom/aomcx.h>
  40 # include <aom/aom_image.h>
  41 # define SOUT_CFG_PREFIX "sout-aom-"
  42 #endif
  43
  44 #include "../packetizer/iso_color_tables.h"
  45
  46 /****************************************************************************
  47  * Local prototypes
  48  ****************************************************************************/
  49 static int OpenDecoder(vlc_object_t *);
  50 static void CloseDecoder(vlc_object_t *);
  51 #ifdef ENABLE_SOUT
  52 static int OpenEncoder(vlc_object_t *);
  53 static void CloseEncoder(vlc_object_t *);
  54 static block_t *Encode(encoder_t *p_enc, picture_t *p_pict);
  55
  56 static const int pi_enc_bitdepth_values_list[] =
  57   { 8, 10, 12 };
  58 static const char *const ppsz_enc_bitdepth_text [] =
  59   { N_("8 bpp"), N_("10 bpp"), N_("12 bpp") };
  60 #endif
  61
  62 /*****************************************************************************
  63  * Module descriptor
  64  *****************************************************************************/
  65
  66 vlc_module_begin ()
  67     set_shortname("aom")
  68     set_description(N_("AOM video decoder"))
  69     set_capability("video decoder", 100)
  70     set_callbacks(OpenDecoder, CloseDecoder)
  71     set_category(CAT_INPUT)
  72     set_subcategory(SUBCAT_INPUT_VCODEC)
  73 #ifdef ENABLE_SOUT
  74     add_submodule()
  75         set_shortname("aom")
  76         set_capability("encoder", 101)
  77         set_description(N_("AOM video encoder"))
  78         set_callbacks(OpenEncoder, CloseEncoder)
  79         add_integer( SOUT_CFG_PREFIX "profile", 0, "Profile", NULL, true )
  80             change_integer_range( 0, 3 )
  81         add_integer( SOUT_CFG_PREFIX "bitdepth", 8, "Bit Depth", NULL, true )
  82             change_integer_list( pi_enc_bitdepth_values_list, ppsz_enc_bitdepth_text )
  83         add_integer( SOUT_CFG_PREFIX "tile-rows", 0, "Tile Rows (in log2 units)", NULL, true )
  84             change_integer_range( 0, 6 ) /* 1 << 6 == MAX_TILE_ROWS */
  85         add_integer( SOUT_CFG_PREFIX "tile-columns", 0, "Tile Columns (in log2 units)", NULL, true )
  86             change_integer_range( 0, 6 ) /* 1 << 6 == MAX_TILE_COLS */
  87 #endif
  88 vlc_module_end ()
  89
  90 static void aom_err_msg(vlc_object_t *this, aom_codec_ctx_t *ctx,
  91                         const char *msg)
  92 {
  93     const char *error  = aom_codec_error(ctx);
  94     const char *detail = aom_codec_error_detail(ctx);
  95     if (!detail)
  96         detail = "no specific information";
  97     msg_Err(this, msg, error, detail);
  98 }
  99
 100 #define AOM_ERR(this, ctx, msg) aom_err_msg(VLC_OBJECT(this), ctx, msg ": %s (%s)")
 101 #define AOM_MAX_FRAMES_DEPTH 64
 102
 103 /*****************************************************************************
 104  * decoder_sys_t: libaom decoder descriptor
 105  *****************************************************************************/
 106 struct frame_priv_s
 107 {
 108     vlc_tick_t pts;
 109 };
 110
 111 typedef struct
 112 {
 113     aom_codec_ctx_t ctx;
 114     struct frame_priv_s frame_priv[AOM_MAX_FRAMES_DEPTH];
 115     unsigned i_next_frame_priv;
 116 } decoder_sys_t;
 117
 118 static const struct
 119 {
 120     vlc_fourcc_t     i_chroma;
 121     enum aom_img_fmt i_chroma_id;
 122     uint8_t          i_bitdepth;
 123     uint8_t          i_needs_hack;
 124
 125 } chroma_table[] =
 126 {
 127     { VLC_CODEC_I420, AOM_IMG_FMT_I420, 8, 0 },
 128     { VLC_CODEC_I422, AOM_IMG_FMT_I422, 8, 0 },
 129     { VLC_CODEC_I444, AOM_IMG_FMT_I444, 8, 0 },
 130
 131     { VLC_CODEC_YV12, AOM_IMG_FMT_YV12, 8, 0 },
 132     { VLC_CODEC_YUVA, AOM_IMG_FMT_444A, 8, 0 },
 133
 134     { VLC_CODEC_GBR_PLANAR, AOM_IMG_FMT_I444, 8, 1 },
 135     { VLC_CODEC_GBR_PLANAR_10L, AOM_IMG_FMT_I44416, 10, 1 },
 136
 137     { VLC_CODEC_I420_10L, AOM_IMG_FMT_I42016, 10, 0 },
 138     { VLC_CODEC_I422_10L, AOM_IMG_FMT_I42216, 10, 0 },
 139     { VLC_CODEC_I444_10L, AOM_IMG_FMT_I44416, 10, 0 },
 140
 141     { VLC_CODEC_I420_12L, AOM_IMG_FMT_I42016, 12, 0 },
 142     { VLC_CODEC_I422_12L, AOM_IMG_FMT_I42216, 12, 0 },
 143     { VLC_CODEC_I444_12L, AOM_IMG_FMT_I44416, 12, 0 },
 144
 145     { VLC_CODEC_I444_16L, AOM_IMG_FMT_I44416, 16, 0 },
 146 };
 147
 148 static vlc_fourcc_t FindVlcChroma( struct aom_image *img )
 149 {
 150     uint8_t hack = (img->fmt & AOM_IMG_FMT_I444) && (img->tc == AOM_CICP_TC_SRGB);
 151
 152     for( unsigned int i = 0; i < ARRAY_SIZE(chroma_table); i++ )
 153         if( chroma_table[i].i_chroma_id == img->fmt &&
 154             chroma_table[i].i_bitdepth == img->bit_depth &&
 155             chroma_table[i].i_needs_hack == hack )
 156             return chroma_table[i].i_chroma;
 157
 158     return 0;
 159 }
 160
 161 static void CopyPicture(const struct aom_image *img, picture_t *pic)
 162 {
 163     for (int plane = 0; plane < pic->i_planes; plane++ ) {
 164         plane_t src_plane = pic->p[plane];
 165         src_plane.p_pixels = img->planes[plane];
 166         src_plane.i_pitch = img->stride[plane];
 167         plane_CopyPixels(&pic->p[plane], &src_plane);
 168     }
 169 }
 170
 171 static int PushFrame(decoder_t *dec, block_t *block)
 172 {
 173     decoder_sys_t *p_sys = dec->p_sys;
 174     aom_codec_ctx_t *ctx = &p_sys->ctx;
 175     const uint8_t *p_buffer;
 176     size_t i_buffer;
 177
 178     /* Associate packet PTS with decoded frame */
 179     uintptr_t priv_index = p_sys->i_next_frame_priv++ % AOM_MAX_FRAMES_DEPTH;
 180
 181     if(likely(block))
 182     {
 183         p_buffer = block->p_buffer;
 184         i_buffer = block->i_buffer;
 185         p_sys->frame_priv[priv_index].pts = (block->i_pts != VLC_TICK_INVALID) ? block->i_pts : block->i_dts;
 186     }
 187     else
 188     {
 189         p_buffer = NULL;
 190         i_buffer = 0;
 191     }
 192
 193     aom_codec_err_t err;
 194     err = aom_codec_decode(ctx, p_buffer, i_buffer, (void*)priv_index);
 195
 196     if(block)
 197         block_Release(block);
 198
 199     if (err != AOM_CODEC_OK) {
 200         AOM_ERR(dec, ctx, "Failed to decode frame");
 201         if (err == AOM_CODEC_UNSUP_BITSTREAM)
 202             return VLCDEC_ECRITICAL;
 203     }
 204     return VLCDEC_SUCCESS;
 205 }
 206
 207 static void OutputFrame(decoder_t *dec, const struct aom_image *img)
 208 {
 209     video_format_t *v = &dec->fmt_out.video;
 210
 211     if (img->d_w != v->i_visible_width || img->d_h != v->i_visible_height)
 212     {
 213         v->i_visible_width = dec->fmt_out.video.i_width = img->d_w;
 214         v->i_visible_height = dec->fmt_out.video.i_height = img->d_h;
 215     }
 216
 217     if( !dec->fmt_out.video.i_sar_num || !dec->fmt_out.video.i_sar_den )
 218     {
 219         dec->fmt_out.video.i_sar_num = 1;
 220         dec->fmt_out.video.i_sar_den = 1;
 221     }
 222
 223     if(dec->fmt_in.video.primaries == COLOR_PRIMARIES_UNDEF)
 224     {
 225         v->primaries = iso_23001_8_cp_to_vlc_primaries(img->cp);
 226         v->transfer = iso_23001_8_tc_to_vlc_xfer(img->tc);
 227         v->space = iso_23001_8_mc_to_vlc_coeffs(img->mc);
 228         v->color_range = img->range == AOM_CR_FULL_RANGE ? COLOR_RANGE_FULL : COLOR_RANGE_LIMITED;
 229     }
 230
 231     dec->fmt_out.video.projection_mode = dec->fmt_in.video.projection_mode;
 232     dec->fmt_out.video.multiview_mode = dec->fmt_in.video.multiview_mode;
 233     dec->fmt_out.video.pose = dec->fmt_in.video.pose;
 234
 235     if (decoder_UpdateVideoFormat(dec) == VLC_SUCCESS)
 236     {
 237         picture_t *pic = decoder_NewPicture(dec);
 238         if (pic)
 239         {
 240             decoder_sys_t *p_sys = dec->p_sys;
 241             CopyPicture(img, pic);
 242
 243             /* fetches back the PTS */
 244
 245             pic->b_progressive = true; /* codec does not support interlacing */
 246             pic->date = p_sys->frame_priv[(uintptr_t)img->user_priv].pts;
 247
 248             decoder_QueueVideo(dec, pic);
 249         }
 250     }
 251 }
 252
 253 static int PopFrames(decoder_t *dec,
 254                      void(*pf_output)(decoder_t *, const struct aom_image *))
 255 {
 256     decoder_sys_t *p_sys = dec->p_sys;
 257     aom_codec_ctx_t *ctx = &p_sys->ctx;
 258
 259     for(const void *iter = NULL;; )
 260     {
 261         struct aom_image *img = aom_codec_get_frame(ctx, &iter);
 262         if (!img)
 263             break;
 264
 265         dec->fmt_out.i_codec = FindVlcChroma(img);
 266         if (dec->fmt_out.i_codec == 0) {
 267             msg_Err(dec, "Unsupported output colorspace %d", img->fmt);
 268             continue;
 269         }
 270
 271         pf_output(dec, img);
 272     }
 273
 274     return VLCDEC_SUCCESS;
 275 }
 276
 277 /****************************************************************************
 278  * Flush: clears decoder between seeks
 279  ****************************************************************************/
 280 static void DropFrame(decoder_t *dec, const struct aom_image *img)
 281 {
 282     VLC_UNUSED(dec);
 283     VLC_UNUSED(img);
 284     /* do nothing for now */
 285 }
 286
 287 static void FlushDecoder(decoder_t *dec)
 288 {
 289     decoder_sys_t *p_sys = dec->p_sys;
 290     aom_codec_ctx_t *ctx = &p_sys->ctx;
 291
 292     if(PushFrame(dec, NULL) != VLCDEC_SUCCESS)
 293         AOM_ERR(dec, ctx, "Failed to flush decoder");
 294     else
 295         PopFrames(dec, DropFrame);
 296 }
 297
 298 /****************************************************************************
 299  * Decode: the whole thing
 300  ****************************************************************************/
 301 static int Decode(decoder_t *dec, block_t *block)
 302 {
 303     if (block && block->i_flags & (BLOCK_FLAG_CORRUPTED))
 304     {
 305         block_Release(block);
 306         return VLCDEC_SUCCESS;
 307     }
 308
 309     int i_ret = PushFrame(dec, block);
 310
 311     PopFrames(dec, OutputFrame);
 312
 313     return i_ret;
 314 }
 315
 316 /*****************************************************************************
 317  * OpenDecoder: probe the decoder
 318  *****************************************************************************/
 319 static int OpenDecoder(vlc_object_t *p_this)
 320 {
 321     decoder_t *dec = (decoder_t *)p_this;
 322     const aom_codec_iface_t *iface;
 323     int av_version;
 324
 325     if (dec->fmt_in.i_codec != VLC_CODEC_AV1)
 326         return VLC_EGENERIC;
 327
 328     iface = &aom_codec_av1_dx_algo;
 329     av_version = 1;
 330
 331     decoder_sys_t *sys = malloc(sizeof(*sys));
 332     if (!sys)
 333         return VLC_ENOMEM;
 334     dec->p_sys = sys;
 335
 336     sys->i_next_frame_priv = 0;
 337
 338     struct aom_codec_dec_cfg deccfg = {
 339         .threads = __MIN(vlc_GetCPUCount(), 16),
 340         .allow_lowbitdepth = 1
 341     };
 342
 343     msg_Dbg(p_this, "AV%d: using libaom version %s (build options %s)",
 344         av_version, aom_codec_version_str(), aom_codec_build_config());
 345
 346     if (aom_codec_dec_init(&sys->ctx, iface, &deccfg, 0) != AOM_CODEC_OK) {
 347         AOM_ERR(p_this, &sys->ctx, "Failed to initialize decoder");
 348         free(sys);
 349         return VLC_EGENERIC;;
 350     }
 351
 352     dec->pf_decode = Decode;
 353     dec->pf_flush = FlushDecoder;
 354
 355     dec->fmt_out.video.i_width = dec->fmt_in.video.i_width;
 356     dec->fmt_out.video.i_height = dec->fmt_in.video.i_height;
 357     dec->fmt_out.i_codec = VLC_CODEC_I420;
 358
 359     if (dec->fmt_in.video.i_sar_num > 0 && dec->fmt_in.video.i_sar_den > 0) {
 360         dec->fmt_out.video.i_sar_num = dec->fmt_in.video.i_sar_num;
 361         dec->fmt_out.video.i_sar_den = dec->fmt_in.video.i_sar_den;
 362     }
 363
 364     return VLC_SUCCESS;
 365 }
 366
 367 static void destroy_context(vlc_object_t *p_this, aom_codec_ctx_t *context)
 368 {
 369     if (aom_codec_destroy(context))
 370         AOM_ERR(p_this, context, "Failed to destroy codec context");
 371 }
 372
 373 /*****************************************************************************
 374  * CloseDecoder: decoder destruction
 375  *****************************************************************************/
 376 static void CloseDecoder(vlc_object_t *p_this)
 377 {
 378     decoder_t *dec = (decoder_t *)p_this;
 379     decoder_sys_t *sys = dec->p_sys;
 380
 381     /* Flush decoder */
 382     FlushDecoder(dec);
 383
 384     destroy_context(p_this, &sys->ctx);
 385
 386     free(sys);
 387 }
 388
 389 #ifdef ENABLE_SOUT
 390
 391 /*****************************************************************************
 392  * encoder_sys_t: libaom encoder descriptor
 393  *****************************************************************************/
 394 typedef struct
 395 {
 396     struct aom_codec_ctx ctx;
 397 } encoder_sys_t;
 398
 399 /*****************************************************************************
 400  * OpenEncoder: probe the encoder
 401  *****************************************************************************/
 402 static int OpenEncoder(vlc_object_t *p_this)
 403 {
 404     encoder_t *p_enc = (encoder_t *)p_this;
 405     encoder_sys_t *p_sys;
 406
 407     if (p_enc->fmt_out.i_codec != VLC_CODEC_AV1)
 408         return VLC_EGENERIC;
 409
 410     /* Allocate the memory needed to store the encoder's structure */
 411     p_sys = malloc(sizeof(*p_sys));
 412     if (p_sys == NULL)
 413         return VLC_ENOMEM;
 414
 415     p_enc->p_sys = p_sys;
 416
 417     const struct aom_codec_iface *iface = &aom_codec_av1_cx_algo;
 418
 419     struct aom_codec_enc_cfg enccfg = {};
 420     aom_codec_enc_config_default(iface, &enccfg, 0);
 421     enccfg.g_timebase.num = p_enc->fmt_in.video.i_frame_rate_base;
 422     enccfg.g_timebase.den = p_enc->fmt_in.video.i_frame_rate;
 423     enccfg.g_threads = __MIN(vlc_GetCPUCount(), 4);
 424     enccfg.g_w = p_enc->fmt_in.video.i_visible_width;
 425     enccfg.g_h = p_enc->fmt_in.video.i_visible_height;
 426
 427     int enc_flags;
 428     int i_profile = var_InheritInteger( p_enc, SOUT_CFG_PREFIX "profile" );
 429     int i_bit_depth = var_InheritInteger( p_enc, SOUT_CFG_PREFIX "bitdepth" );
 430     int i_tile_rows = var_InheritInteger( p_enc, SOUT_CFG_PREFIX "tile-rows" );
 431     int i_tile_columns = var_InheritInteger( p_enc, SOUT_CFG_PREFIX "tile-columns" );
 432
 433     /* TODO: implement higher profiles, bit depths and other pixformats. */
 434     switch( i_profile )
 435     {
 436         case 0:
 437             /* Main Profile: 8 and 10-bit 4:2:0. */
 438             enccfg.g_profile = 0;
 439             switch( i_bit_depth )
 440             {
 441                 case 10:
 442                     p_enc->fmt_in.i_codec = VLC_CODEC_I420_10L;
 443                     enc_flags = AOM_CODEC_USE_HIGHBITDEPTH;
 444                     break;
 445                 case 8:
 446                     p_enc->fmt_in.i_codec = VLC_CODEC_I420;
 447                     enc_flags = 0;
 448                     break;
 449                 default:
 450                     msg_Err( p_enc, "%d bit is unsupported for profile %d", i_bit_depth, i_profile );
 451                     free( p_sys );
 452                     return VLC_EGENERIC;
 453             }
 454             enccfg.g_bit_depth = i_bit_depth;
 455             break;
 456
 457         case 1:
 458             /* High Profile: 8 and 10-bit 4:4:4 */
 459             /* fallthrough */
 460         case 2:
 461             /* Professional Profile: 8, 10 and 12-bit for 4:2:2, otherwise 12-bit. */
 462             /* fallthrough */
 463         default:
 464             msg_Err( p_enc, "Unsupported profile %d", i_profile );
 465             free( p_sys );
 466             return VLC_EGENERIC;
 467     }
 468
 469     msg_Dbg(p_this, "AV1: using libaom version %s (build options %s)",
 470         aom_codec_version_str(), aom_codec_build_config());
 471
 472     struct aom_codec_ctx *ctx = &p_sys->ctx;
 473     if (aom_codec_enc_init(ctx, iface, &enccfg, enc_flags) != AOM_CODEC_OK)
 474     {
 475         AOM_ERR(p_this, ctx, "Failed to initialize encoder");
 476         free(p_sys);
 477         return VLC_EGENERIC;
 478     }
 479
 480     if (i_tile_rows >= 0 &&
 481         aom_codec_control(ctx, AV1E_SET_TILE_ROWS, i_tile_rows))
 482     {
 483         AOM_ERR(p_this, ctx, "Failed to set tile rows");
 484         destroy_context(p_this, ctx);
 485         free(p_sys);
 486         return VLC_EGENERIC;
 487     }
 488
 489     if (i_tile_columns >= 0 &&
 490         aom_codec_control(ctx, AV1E_SET_TILE_COLUMNS, i_tile_columns))
 491     {
 492         AOM_ERR(p_this, ctx, "Failed to set tile columns");
 493         destroy_context(p_this, ctx);
 494         free(p_sys);
 495         return VLC_EGENERIC;
 496     }
 497
 498     p_enc->pf_encode_video = Encode;
 499
 500     return VLC_SUCCESS;
 501 }
 502
 503 /****************************************************************************
 504  * Encode: the whole thing
 505  ****************************************************************************/
 506 static block_t *Encode(encoder_t *p_enc, picture_t *p_pict)
 507 {
 508     encoder_sys_t *p_sys = p_enc->p_sys;
 509     struct aom_codec_ctx *ctx = &p_sys->ctx;
 510
 511     if (!p_pict) return NULL;
 512
 513     aom_image_t img = {};
 514     unsigned i_w = p_enc->fmt_in.video.i_visible_width;
 515     unsigned i_h = p_enc->fmt_in.video.i_visible_height;
 516     const aom_img_fmt_t img_fmt = p_enc->fmt_in.i_codec == VLC_CODEC_I420_10L ?
 517         AOM_IMG_FMT_I42016 : AOM_IMG_FMT_I420;
 518
 519     /* Create and initialize the aom_image */
 520     if (!aom_img_wrap(&img, img_fmt, i_w, i_h, 32, p_pict->p[0].p_pixels))
 521     {
 522         AOM_ERR(p_enc, ctx, "Failed to wrap image");
 523         return NULL;
 524     }
 525
 526     /* Correct chroma plane offsets. */
 527     for (int plane = 1; plane < p_pict->i_planes; plane++) {
 528         img.planes[plane] = p_pict->p[plane].p_pixels;
 529         img.stride[plane] = p_pict->p[plane].i_pitch;
 530     }
 531
 532     aom_codec_err_t res = aom_codec_encode(ctx, &img, US_FROM_VLC_TICK(p_pict->date), 1, 0);
 533     if (res != AOM_CODEC_OK) {
 534         AOM_ERR(p_enc, ctx, "Failed to encode frame");
 535         aom_img_free(&img);
 536         return NULL;
 537     }
 538
 539     const aom_codec_cx_pkt_t *pkt = NULL;
 540     aom_codec_iter_t iter = NULL;
 541     block_t *p_out = NULL;
 542     while ((pkt = aom_codec_get_cx_data(ctx, &iter)) != NULL)
 543     {
 544         if (pkt->kind == AOM_CODEC_CX_FRAME_PKT)
 545         {
 546             int keyframe = pkt->data.frame.flags & AOM_FRAME_IS_KEY;
 547             block_t *p_block = block_Alloc(pkt->data.frame.sz);
 548             if (unlikely(p_block == NULL)) {
 549                 block_ChainRelease(p_out);
 550                 p_out = NULL;
 551                 break;
 552             }
 553
 554             /* FIXME: do this in-place */
 555             memcpy(p_block->p_buffer, pkt->data.frame.buf, pkt->data.frame.sz);
 556             p_block->i_dts = p_block->i_pts = VLC_TICK_FROM_US(pkt->data.frame.pts);
 557             if (keyframe)
 558                 p_block->i_flags |= BLOCK_FLAG_TYPE_I;
 559             block_ChainAppend(&p_out, p_block);
 560         }
 561     }
 562     aom_img_free(&img);
 563     return p_out;
 564 }
 565
 566 /*****************************************************************************
 567  * CloseEncoder: encoder destruction
 568  *****************************************************************************/
 569 static void CloseEncoder(vlc_object_t *p_this)
 570 {
 571     encoder_t *p_enc = (encoder_t *)p_this;
 572     encoder_sys_t *p_sys = p_enc->p_sys;
 573     destroy_context(p_this, &p_sys->ctx);
 574     free(p_sys);
 575 }
 576
 577 #endif  /* ENABLE_SOUT */