examples/alffplay.cpp

   1 /*
   2  * An example showing how to play a stream sync'd to video, using ffmpeg.
   3  *
   4  * Requires C++11.
   5  */
   6
   7 #include <condition_variable>
   8 #include <functional>
   9 #include <algorithm>
  10 #include <iostream>
  11 #include <iomanip>
  12 #include <cstring>
  13 #include <limits>
  14 #include <thread>
  15 #include <chrono>
  16 #include <atomic>
  17 #include <vector>
  18 #include <mutex>
  19 #include <deque>
  20 #include <array>
  21
  22 extern "C" {
  23 #include "libavcodec/avcodec.h"
  24 #include "libavformat/avformat.h"
  25 #include "libavformat/avio.h"
  26 #include "libavutil/time.h"
  27 #include "libavutil/pixfmt.h"
  28 #include "libavutil/avstring.h"
  29 #include "libavutil/channel_layout.h"
  30 #include "libswscale/swscale.h"
  31 #include "libswresample/swresample.h"
  32 }
  33
  34 #include "SDL.h"
  35
  36 #include "AL/alc.h"
  37 #include "AL/al.h"
  38 #include "AL/alext.h"
  39
  40 extern "C" {
  41 #ifndef AL_SOFT_map_buffer
  42 #define AL_SOFT_map_buffer 1
  43 typedef unsigned int ALbitfieldSOFT;
  44 #define AL_MAP_READ_BIT_SOFT                     0x00000001
  45 #define AL_MAP_WRITE_BIT_SOFT                    0x00000002
  46 #define AL_MAP_PERSISTENT_BIT_SOFT               0x00000004
  47 #define AL_PRESERVE_DATA_BIT_SOFT                0x00000008
  48 typedef void (AL_APIENTRY*LPALBUFFERSTORAGESOFT)(ALuint buffer, ALenum format, const ALvoid *data, ALsizei size, ALsizei freq, ALbitfieldSOFT flags);
  49 typedef void* (AL_APIENTRY*LPALMAPBUFFERSOFT)(ALuint buffer, ALsizei offset, ALsizei length, ALbitfieldSOFT access);
  50 typedef void (AL_APIENTRY*LPALUNMAPBUFFERSOFT)(ALuint buffer);
  51 typedef void (AL_APIENTRY*LPALFLUSHMAPPEDBUFFERSOFT)(ALuint buffer, ALsizei offset, ALsizei length);
  52 #endif
  53 }
  54
  55 namespace {
  56
  57 using nanoseconds = std::chrono::nanoseconds;
  58 using microseconds = std::chrono::microseconds;
  59 using milliseconds = std::chrono::milliseconds;
  60 using seconds = std::chrono::seconds;
  61 using seconds_d64 = std::chrono::duration<double>;
  62
  63 const std::string AppName("alffplay");
  64
  65 bool EnableDirectOut = false;
  66 LPALGETSOURCEI64VSOFT alGetSourcei64vSOFT;
  67 LPALCGETINTEGER64VSOFT alcGetInteger64vSOFT;
  68
  69 LPALBUFFERSTORAGESOFT alBufferStorageSOFT;
  70 LPALMAPBUFFERSOFT alMapBufferSOFT;
  71 LPALUNMAPBUFFERSOFT alUnmapBufferSOFT;
  72
  73 const seconds AVNoSyncThreshold(10);
  74
  75 const milliseconds VideoSyncThreshold(10);
  76 #define VIDEO_PICTURE_QUEUE_SIZE 16
  77
  78 const seconds_d64 AudioSyncThreshold(0.03);
  79 const milliseconds AudioSampleCorrectionMax(50);
  80 /* Averaging filter coefficient for audio sync. */
  81 #define AUDIO_DIFF_AVG_NB 20
  82 const double AudioAvgFilterCoeff = std::pow(0.01, 1.0/AUDIO_DIFF_AVG_NB);
  83 /* Per-buffer size, in time */
  84 const milliseconds AudioBufferTime(20);
  85 /* Buffer total size, in time (should be divisible by the buffer time) */
  86 const milliseconds AudioBufferTotalTime(800);
  87
  88 #define MAX_QUEUE_SIZE (15 * 1024 * 1024) /* Bytes of compressed data to keep queued */
  89
  90 enum {
  91     FF_UPDATE_EVENT = SDL_USEREVENT,
  92     FF_REFRESH_EVENT,
  93     FF_MOVIE_DONE_EVENT
  94 };
  95
  96 enum class SyncMaster {
  97     Audio,
  98     Video,
  99     External,
 100
 101     Default = External
 102 };
 103
 104
 105 inline microseconds get_avtime()
 106 { return microseconds(av_gettime()); }
 107
 108 /* Define unique_ptrs to auto-cleanup associated ffmpeg objects. */
 109 struct AVIOContextDeleter {
 110     void operator()(AVIOContext *ptr) { avio_closep(&ptr); }
 111 };
 112 using AVIOContextPtr = std::unique_ptr<AVIOContext,AVIOContextDeleter>;
 113
 114 struct AVFormatCtxDeleter {
 115     void operator()(AVFormatContext *ptr) { avformat_close_input(&ptr); }
 116 };
 117 using AVFormatCtxPtr = std::unique_ptr<AVFormatContext,AVFormatCtxDeleter>;
 118
 119 struct AVCodecCtxDeleter {
 120     void operator()(AVCodecContext *ptr) { avcodec_free_context(&ptr); }
 121 };
 122 using AVCodecCtxPtr = std::unique_ptr<AVCodecContext,AVCodecCtxDeleter>;
 123
 124 struct AVFrameDeleter {
 125     void operator()(AVFrame *ptr) { av_frame_free(&ptr); }
 126 };
 127 using AVFramePtr = std::unique_ptr<AVFrame,AVFrameDeleter>;
 128
 129 struct SwrContextDeleter {
 130     void operator()(SwrContext *ptr) { swr_free(&ptr); }
 131 };
 132 using SwrContextPtr = std::unique_ptr<SwrContext,SwrContextDeleter>;
 133
 134 struct SwsContextDeleter {
 135     void operator()(SwsContext *ptr) { sws_freeContext(ptr); }
 136 };
 137 using SwsContextPtr = std::unique_ptr<SwsContext,SwsContextDeleter>;
 138
 139
 140 class PacketQueue {
 141     std::deque<AVPacket> mPackets;
 142     size_t mTotalSize{0};
 143
 144 public:
 145     ~PacketQueue() { clear(); }
 146
 147     bool empty() const noexcept { return mPackets.empty(); }
 148     size_t totalSize() const noexcept { return mTotalSize; }
 149
 150     void put(const AVPacket *pkt)
 151     {
 152         mPackets.push_back(AVPacket{});
 153         if(av_packet_ref(&mPackets.back(), pkt) != 0)
 154             mPackets.pop_back();
 155         else
 156             mTotalSize += mPackets.back().size;
 157     }
 158
 159     AVPacket *front() noexcept
 160     { return &mPackets.front(); }
 161
 162     void pop()
 163     {
 164         AVPacket *pkt = &mPackets.front();
 165         mTotalSize -= pkt->size;
 166         av_packet_unref(pkt);
 167         mPackets.pop_front();
 168     }
 169
 170     void clear()
 171     {
 172         for(AVPacket &pkt : mPackets)
 173             av_packet_unref(&pkt);
 174         mPackets.clear();
 175         mTotalSize = 0;
 176     }
 177 };
 178
 179
 180 struct MovieState;
 181
 182 struct AudioState {
 183     MovieState &mMovie;
 184
 185     AVStream *mStream{nullptr};
 186     AVCodecCtxPtr mCodecCtx;
 187
 188     std::mutex mQueueMtx;
 189     std::condition_variable mQueueCond;
 190
 191     /* Used for clock difference average computation */
 192     seconds_d64 mClockDiffAvg{0};
 193
 194     /* Time of the next sample to be buffered */
 195     nanoseconds mCurrentPts{0};
 196
 197     /* Device clock time that the stream started at. */
 198     nanoseconds mDeviceStartTime{nanoseconds::min()};
 199
 200     /* Decompressed sample frame, and swresample context for conversion */
 201     AVFramePtr    mDecodedFrame;
 202     SwrContextPtr mSwresCtx;
 203
 204     /* Conversion format, for what gets fed to OpenAL */
 205     int            mDstChanLayout{0};
 206     AVSampleFormat mDstSampleFmt{AV_SAMPLE_FMT_NONE};
 207
 208     /* Storage of converted samples */
 209     uint8_t *mSamples{nullptr};
 210     int mSamplesLen{0}; /* In samples */
 211     int mSamplesPos{0};
 212     int mSamplesMax{0};
 213
 214     /* OpenAL format */
 215     ALenum mFormat{AL_NONE};
 216     ALsizei mFrameSize{0};
 217
 218     std::mutex mSrcMutex;
 219     ALuint mSource{0};
 220     std::vector<ALuint> mBuffers;
 221     ALsizei mBufferIdx{0};
 222
 223     AudioState(MovieState &movie) : mMovie(movie)
 224     { }
 225     ~AudioState()
 226     {
 227         if(mSource)
 228             alDeleteSources(1, &mSource);
 229         if(!mBuffers.empty())
 230             alDeleteBuffers(mBuffers.size(), mBuffers.data());
 231
 232         av_freep(&mSamples);
 233     }
 234
 235     nanoseconds getClockNoLock();
 236     nanoseconds getClock()
 237     {
 238         std::lock_guard<std::mutex> lock(mSrcMutex);
 239         return getClockNoLock();
 240     }
 241
 242     bool isBufferFilled();
 243     void startPlayback();
 244
 245     int getSync();
 246     int decodeFrame();
 247     bool readAudio(uint8_t *samples, int length);
 248
 249     int handler();
 250 };
 251
 252 struct VideoState {
 253     MovieState &mMovie;
 254
 255     AVStream *mStream{nullptr};
 256     AVCodecCtxPtr mCodecCtx;
 257
 258     std::mutex mQueueMtx;
 259     std::condition_variable mQueueCond;
 260
 261     nanoseconds mClock{0};
 262     nanoseconds mFrameTimer{0};
 263     nanoseconds mFrameLastPts{0};
 264     nanoseconds mFrameLastDelay{0};
 265     nanoseconds mCurrentPts{0};
 266     /* time (av_gettime) at which we updated mCurrentPts - used to have running video pts */
 267     microseconds mCurrentPtsTime{0};
 268
 269     /* Decompressed video frame, and swscale context for conversion */
 270     AVFramePtr    mDecodedFrame;
 271     SwsContextPtr mSwscaleCtx;
 272
 273     struct Picture {
 274         SDL_Texture *mImage{nullptr};
 275         int mWidth{0}, mHeight{0}; /* Logical image size (actual size may be larger) */
 276         std::atomic<bool> mUpdated{false};
 277         nanoseconds mPts{0};
 278
 279         ~Picture()
 280         {
 281             if(mImage)
 282                 SDL_DestroyTexture(mImage);
 283             mImage = nullptr;
 284         }
 285     };
 286     std::array<Picture,VIDEO_PICTURE_QUEUE_SIZE> mPictQ;
 287     size_t mPictQSize{0}, mPictQRead{0}, mPictQWrite{0};
 288     std::mutex mPictQMutex;
 289     std::condition_variable mPictQCond;
 290     bool mFirstUpdate{true};
 291     std::atomic<bool> mEOS{false};
 292     std::atomic<bool> mFinalUpdate{false};
 293
 294     VideoState(MovieState &movie) : mMovie(movie) { }
 295
 296     nanoseconds getClock();
 297     bool isBufferFilled();
 298
 299     static Uint32 SDLCALL sdl_refresh_timer_cb(Uint32 interval, void *opaque);
 300     void schedRefresh(milliseconds delay);
 301     void display(SDL_Window *screen, SDL_Renderer *renderer);
 302     void refreshTimer(SDL_Window *screen, SDL_Renderer *renderer);
 303     void updatePicture(SDL_Window *screen, SDL_Renderer *renderer);
 304     int queuePicture(nanoseconds pts);
 305     int handler();
 306 };
 307
 308 struct MovieState {
 309     AVIOContextPtr mIOContext;
 310     AVFormatCtxPtr mFormatCtx;
 311
 312     SyncMaster mAVSyncType{SyncMaster::Default};
 313
 314     microseconds mClockBase{0};
 315     std::atomic<bool> mPlaying{false};
 316
 317     std::mutex mSendMtx;
 318     std::condition_variable mSendCond;
 319     /* NOTE: false/clear = need data, true/set = no data needed */
 320     std::atomic_flag mSendDataGood;
 321
 322     std::atomic<bool> mQuit{false};
 323
 324     AudioState mAudio;
 325     VideoState mVideo;
 326
 327     std::thread mParseThread;
 328     std::thread mAudioThread;
 329     std::thread mVideoThread;
 330
 331     std::string mFilename;
 332
 333     MovieState(std::string fname)
 334       : mAudio(*this), mVideo(*this), mFilename(std::move(fname))
 335     { }
 336     ~MovieState()
 337     {
 338         mQuit = true;
 339         if(mParseThread.joinable())
 340             mParseThread.join();
 341     }
 342
 343     static int decode_interrupt_cb(void *ctx);
 344     bool prepare();
 345     void setTitle(SDL_Window *window);
 346
 347     nanoseconds getClock();
 348
 349     nanoseconds getMasterClock();
 350
 351     nanoseconds getDuration();
 352
 353     int streamComponentOpen(int stream_index);
 354     int parse_handler();
 355 };
 356
 357
 358 nanoseconds AudioState::getClockNoLock()
 359 {
 360     // The audio clock is the timestamp of the sample currently being heard.
 361     if(alcGetInteger64vSOFT)
 362     {
 363         // If device start time = min, we aren't playing yet.
 364         if(mDeviceStartTime == nanoseconds::min())
 365             return nanoseconds::zero();
 366
 367         // Get the current device clock time and latency.
 368         auto device = alcGetContextsDevice(alcGetCurrentContext());
 369         ALCint64SOFT devtimes[2] = {0,0};
 370         alcGetInteger64vSOFT(device, ALC_DEVICE_CLOCK_LATENCY_SOFT, 2, devtimes);
 371         auto latency = nanoseconds(devtimes[1]);
 372         auto device_time = nanoseconds(devtimes[0]);
 373
 374         // The clock is simply the current device time relative to the recorded
 375         // start time. We can also subtract the latency to get more a accurate
 376         // position of where the audio device actually is in the output stream.
 377         return device_time - mDeviceStartTime - latency;
 378     }
 379
 380     /* The source-based clock is based on 4 components:
 381      * 1 - The timestamp of the next sample to buffer (mCurrentPts)
 382      * 2 - The length of the source's buffer queue
 383      *     (AudioBufferTime*AL_BUFFERS_QUEUED)
 384      * 3 - The offset OpenAL is currently at in the source (the first value
 385      *     from AL_SAMPLE_OFFSET_LATENCY_SOFT)
 386      * 4 - The latency between OpenAL and the DAC (the second value from
 387      *     AL_SAMPLE_OFFSET_LATENCY_SOFT)
 388      *
 389      * Subtracting the length of the source queue from the next sample's
 390      * timestamp gives the timestamp of the sample at the start of the source
 391      * queue. Adding the source offset to that results in the timestamp for the
 392      * sample at OpenAL's current position, and subtracting the source latency
 393      * from that gives the timestamp of the sample currently at the DAC.
 394      */
 395     nanoseconds pts = mCurrentPts;
 396     if(mSource)
 397     {
 398         ALint64SOFT offset[2];
 399         ALint queued;
 400         ALint status;
 401
 402         /* NOTE: The source state must be checked last, in case an underrun
 403          * occurs and the source stops between retrieving the offset+latency
 404          * and getting the state. */
 405         if(alGetSourcei64vSOFT)
 406             alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_LATENCY_SOFT, offset);
 407         else
 408         {
 409             ALint ioffset;
 410             alGetSourcei(mSource, AL_SAMPLE_OFFSET, &ioffset);
 411             offset[0] = (ALint64SOFT)ioffset << 32;
 412             offset[1] = 0;
 413         }
 414         alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
 415         alGetSourcei(mSource, AL_SOURCE_STATE, &status);
 416
 417         /* If the source is AL_STOPPED, then there was an underrun and all
 418          * buffers are processed, so ignore the source queue. The audio thread
 419          * will put the source into an AL_INITIAL state and clear the queue
 420          * when it starts recovery. */
 421         if(status != AL_STOPPED)
 422         {
 423             using fixed32 = std::chrono::duration<int64_t,std::ratio<1,(1ll<<32)>>;
 424
 425             pts -= AudioBufferTime*queued;
 426             pts += std::chrono::duration_cast<nanoseconds>(
 427                 fixed32(offset[0] / mCodecCtx->sample_rate)
 428             );
 429         }
 430         /* Don't offset by the latency if the source isn't playing. */
 431         if(status == AL_PLAYING)
 432             pts -= nanoseconds(offset[1]);
 433     }
 434
 435     return std::max(pts, nanoseconds::zero());
 436 }
 437
 438 bool AudioState::isBufferFilled()
 439 {
 440     /* All of OpenAL's buffer queueing happens under the mSrcMutex lock, as
 441      * does the source gen. So when we're able to grab the lock and the source
 442      * is valid, the queue must be full.
 443      */
 444     std::lock_guard<std::mutex> lock(mSrcMutex);
 445     return mSource != 0;
 446 }
 447
 448 void AudioState::startPlayback()
 449 {
 450     alSourcePlay(mSource);
 451     if(alcGetInteger64vSOFT)
 452     {
 453         using fixed32 = std::chrono::duration<int64_t,std::ratio<1,(1ll<<32)>>;
 454
 455         // Subtract the total buffer queue time from the current pts to get the
 456         // pts of the start of the queue.
 457         nanoseconds startpts = mCurrentPts - AudioBufferTotalTime;
 458         int64_t srctimes[2]={0,0};
 459         alGetSourcei64vSOFT(mSource, AL_SAMPLE_OFFSET_CLOCK_SOFT, srctimes);
 460         auto device_time = nanoseconds(srctimes[1]);
 461         auto src_offset = std::chrono::duration_cast<nanoseconds>(fixed32(srctimes[0])) /
 462                           mCodecCtx->sample_rate;
 463
 464         // The mixer may have ticked and incremented the device time and sample
 465         // offset, so subtract the source offset from the device time to get
 466         // the device time the source started at. Also subtract startpts to get
 467         // the device time the stream would have started at to reach where it
 468         // is now.
 469         mDeviceStartTime = device_time - src_offset - startpts;
 470     }
 471 }
 472
 473 int AudioState::getSync()
 474 {
 475     if(mMovie.mAVSyncType == SyncMaster::Audio)
 476         return 0;
 477
 478     auto ref_clock = mMovie.getMasterClock();
 479     auto diff = ref_clock - getClockNoLock();
 480
 481     if(!(diff < AVNoSyncThreshold && diff > -AVNoSyncThreshold))
 482     {
 483         /* Difference is TOO big; reset accumulated average */
 484         mClockDiffAvg = seconds_d64::zero();
 485         return 0;
 486     }
 487
 488     /* Accumulate the diffs */
 489     mClockDiffAvg = mClockDiffAvg*AudioAvgFilterCoeff + diff;
 490     auto avg_diff = mClockDiffAvg*(1.0 - AudioAvgFilterCoeff);
 491     if(avg_diff < AudioSyncThreshold/2.0 && avg_diff > -AudioSyncThreshold)
 492         return 0;
 493
 494     /* Constrain the per-update difference to avoid exceedingly large skips */
 495     diff = std::min<nanoseconds>(std::max<nanoseconds>(diff, -AudioSampleCorrectionMax),
 496                                  AudioSampleCorrectionMax);
 497     return (int)std::chrono::duration_cast<seconds>(diff*mCodecCtx->sample_rate).count();
 498 }
 499
 500 int AudioState::decodeFrame()
 501 {
 502     while(!mMovie.mQuit.load(std::memory_order_relaxed))
 503     {
 504         std::unique_lock<std::mutex> lock(mQueueMtx);
 505         int ret = avcodec_receive_frame(mCodecCtx.get(), mDecodedFrame.get());
 506         if(ret == AVERROR(EAGAIN))
 507         {
 508             mMovie.mSendDataGood.clear(std::memory_order_relaxed);
 509             std::unique_lock<std::mutex>(mMovie.mSendMtx).unlock();
 510             mMovie.mSendCond.notify_one();
 511             do {
 512                 mQueueCond.wait(lock);
 513                 ret = avcodec_receive_frame(mCodecCtx.get(), mDecodedFrame.get());
 514             } while(ret == AVERROR(EAGAIN));
 515         }
 516         lock.unlock();
 517         if(ret == AVERROR_EOF) break;
 518         mMovie.mSendDataGood.clear(std::memory_order_relaxed);
 519         mMovie.mSendCond.notify_one();
 520         if(ret < 0)
 521         {
 522             std::cerr<< "Failed to decode frame: "<<ret <<std::endl;
 523             return 0;
 524         }
 525
 526         if(mDecodedFrame->nb_samples <= 0)
 527         {
 528             av_frame_unref(mDecodedFrame.get());
 529             continue;
 530         }
 531
 532         /* If provided, update w/ pts */
 533         if(mDecodedFrame->best_effort_timestamp != AV_NOPTS_VALUE)
 534             mCurrentPts = std::chrono::duration_cast<nanoseconds>(
 535                 seconds_d64(av_q2d(mStream->time_base)*mDecodedFrame->best_effort_timestamp)
 536             );
 537
 538         if(mDecodedFrame->nb_samples > mSamplesMax)
 539         {
 540             av_freep(&mSamples);
 541             av_samples_alloc(
 542                 &mSamples, nullptr, mCodecCtx->channels,
 543                 mDecodedFrame->nb_samples, mDstSampleFmt, 0
 544             );
 545             mSamplesMax = mDecodedFrame->nb_samples;
 546         }
 547         /* Return the amount of sample frames converted */
 548         int data_size = swr_convert(mSwresCtx.get(), &mSamples, mDecodedFrame->nb_samples,
 549             (const uint8_t**)mDecodedFrame->data, mDecodedFrame->nb_samples
 550         );
 551
 552         av_frame_unref(mDecodedFrame.get());
 553         return data_size;
 554     }
 555
 556     return 0;
 557 }
 558
 559 /* Duplicates the sample at in to out, count times. The frame size is a
 560  * multiple of the template type size.
 561  */
 562 template<typename T>
 563 static void sample_dup(uint8_t *out, const uint8_t *in, int count, int frame_size)
 564 {
 565     const T *sample = reinterpret_cast<const T*>(in);
 566     T *dst = reinterpret_cast<T*>(out);
 567     if(frame_size == sizeof(T))
 568         std::fill_n(dst, count, *sample);
 569     else
 570     {
 571         /* NOTE: frame_size is a multiple of sizeof(T). */
 572         int type_mult = frame_size / sizeof(T);
 573         int i = 0;
 574         std::generate_n(dst, count*type_mult,
 575             [sample,type_mult,&i]() -> T
 576             {
 577                 T ret = sample[i];
 578                 i = (i+1)%type_mult;
 579                 return ret;
 580             }
 581         );
 582     }
 583 }
 584
 585
 586 bool AudioState::readAudio(uint8_t *samples, int length)
 587 {
 588     int sample_skip = getSync();
 589     int audio_size = 0;
 590
 591     /* Read the next chunk of data, refill the buffer, and queue it
 592      * on the source */
 593     length /= mFrameSize;
 594     while(audio_size < length)
 595     {
 596         if(mSamplesLen <= 0 || mSamplesPos >= mSamplesLen)
 597         {
 598             int frame_len = decodeFrame();
 599             if(frame_len <= 0) break;
 600
 601             mSamplesLen = frame_len;
 602             mSamplesPos = std::min(mSamplesLen, sample_skip);
 603             sample_skip -= mSamplesPos;
 604
 605             // Adjust the device start time and current pts by the amount we're
 606             // skipping/duplicating, so that the clock remains correct for the
 607             // current stream position.
 608             auto skip = nanoseconds(seconds(mSamplesPos)) / mCodecCtx->sample_rate;
 609             mDeviceStartTime -= skip;
 610             mCurrentPts += skip;
 611             continue;
 612         }
 613
 614         int rem = length - audio_size;
 615         if(mSamplesPos >= 0)
 616         {
 617             int len = mSamplesLen - mSamplesPos;
 618             if(rem > len) rem = len;
 619             memcpy(samples, mSamples + mSamplesPos*mFrameSize, rem*mFrameSize);
 620         }
 621         else
 622         {
 623             rem = std::min(rem, -mSamplesPos);
 624
 625             /* Add samples by copying the first sample */
 626             if((mFrameSize&7) == 0)
 627                 sample_dup<uint64_t>(samples, mSamples, rem, mFrameSize);
 628             else if((mFrameSize&3) == 0)
 629                 sample_dup<uint32_t>(samples, mSamples, rem, mFrameSize);
 630             else if((mFrameSize&1) == 0)
 631                 sample_dup<uint16_t>(samples, mSamples, rem, mFrameSize);
 632             else
 633                 sample_dup<uint8_t>(samples, mSamples, rem, mFrameSize);
 634         }
 635
 636         mSamplesPos += rem;
 637         mCurrentPts += nanoseconds(seconds(rem)) / mCodecCtx->sample_rate;
 638         samples += rem*mFrameSize;
 639         audio_size += rem;
 640     }
 641     if(audio_size <= 0)
 642         return false;
 643
 644     if(audio_size < length)
 645     {
 646         int rem = length - audio_size;
 647         std::fill_n(samples, rem*mFrameSize,
 648                     (mDstSampleFmt == AV_SAMPLE_FMT_U8) ? 0x80 : 0x00);
 649         mCurrentPts += nanoseconds(seconds(rem)) / mCodecCtx->sample_rate;
 650         audio_size += rem;
 651     }
 652     return true;
 653 }
 654
 655
 656 int AudioState::handler()
 657 {
 658     std::unique_lock<std::mutex> lock(mSrcMutex);
 659     ALenum fmt;
 660
 661     /* Find a suitable format for OpenAL. */
 662     mDstChanLayout = 0;
 663     if(mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8 || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_U8P)
 664     {
 665         mDstSampleFmt = AV_SAMPLE_FMT_U8;
 666         mFrameSize = 1;
 667         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_7POINT1 &&
 668            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
 669            (fmt=alGetEnumValue("AL_FORMAT_71CHN8")) != AL_NONE && fmt != -1)
 670         {
 671             mDstChanLayout = mCodecCtx->channel_layout;
 672             mFrameSize *= 8;
 673             mFormat = fmt;
 674         }
 675         if((mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1 ||
 676             mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1_BACK) &&
 677            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
 678            (fmt=alGetEnumValue("AL_FORMAT_51CHN8")) != AL_NONE && fmt != -1)
 679         {
 680             mDstChanLayout = mCodecCtx->channel_layout;
 681             mFrameSize *= 6;
 682             mFormat = fmt;
 683         }
 684         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_MONO)
 685         {
 686             mDstChanLayout = mCodecCtx->channel_layout;
 687             mFrameSize *= 1;
 688             mFormat = AL_FORMAT_MONO8;
 689         }
 690         if(!mDstChanLayout)
 691         {
 692             mDstChanLayout = AV_CH_LAYOUT_STEREO;
 693             mFrameSize *= 2;
 694             mFormat = AL_FORMAT_STEREO8;
 695         }
 696     }
 697     if((mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLT || mCodecCtx->sample_fmt == AV_SAMPLE_FMT_FLTP) &&
 698        alIsExtensionPresent("AL_EXT_FLOAT32"))
 699     {
 700         mDstSampleFmt = AV_SAMPLE_FMT_FLT;
 701         mFrameSize = 4;
 702         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_7POINT1 &&
 703            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
 704            (fmt=alGetEnumValue("AL_FORMAT_71CHN32")) != AL_NONE && fmt != -1)
 705         {
 706             mDstChanLayout = mCodecCtx->channel_layout;
 707             mFrameSize *= 8;
 708             mFormat = fmt;
 709         }
 710         if((mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1 ||
 711             mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1_BACK) &&
 712            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
 713            (fmt=alGetEnumValue("AL_FORMAT_51CHN32")) != AL_NONE && fmt != -1)
 714         {
 715             mDstChanLayout = mCodecCtx->channel_layout;
 716             mFrameSize *= 6;
 717             mFormat = fmt;
 718         }
 719         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_MONO)
 720         {
 721             mDstChanLayout = mCodecCtx->channel_layout;
 722             mFrameSize *= 1;
 723             mFormat = AL_FORMAT_MONO_FLOAT32;
 724         }
 725         if(!mDstChanLayout)
 726         {
 727             mDstChanLayout = AV_CH_LAYOUT_STEREO;
 728             mFrameSize *= 2;
 729             mFormat = AL_FORMAT_STEREO_FLOAT32;
 730         }
 731     }
 732     if(!mDstChanLayout)
 733     {
 734         mDstSampleFmt = AV_SAMPLE_FMT_S16;
 735         mFrameSize = 2;
 736         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_7POINT1 &&
 737            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
 738            (fmt=alGetEnumValue("AL_FORMAT_71CHN16")) != AL_NONE && fmt != -1)
 739         {
 740             mDstChanLayout = mCodecCtx->channel_layout;
 741             mFrameSize *= 8;
 742             mFormat = fmt;
 743         }
 744         if((mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1 ||
 745             mCodecCtx->channel_layout == AV_CH_LAYOUT_5POINT1_BACK) &&
 746            alIsExtensionPresent("AL_EXT_MCFORMATS") &&
 747            (fmt=alGetEnumValue("AL_FORMAT_51CHN16")) != AL_NONE && fmt != -1)
 748         {
 749             mDstChanLayout = mCodecCtx->channel_layout;
 750             mFrameSize *= 6;
 751             mFormat = fmt;
 752         }
 753         if(mCodecCtx->channel_layout == AV_CH_LAYOUT_MONO)
 754         {
 755             mDstChanLayout = mCodecCtx->channel_layout;
 756             mFrameSize *= 1;
 757             mFormat = AL_FORMAT_MONO16;
 758         }
 759         if(!mDstChanLayout)
 760         {
 761             mDstChanLayout = AV_CH_LAYOUT_STEREO;
 762             mFrameSize *= 2;
 763             mFormat = AL_FORMAT_STEREO16;
 764         }
 765     }
 766     void *samples = nullptr;
 767     ALsizei buffer_len = std::chrono::duration_cast<std::chrono::duration<int>>(
 768         mCodecCtx->sample_rate * AudioBufferTime).count() * mFrameSize;
 769
 770     mSamples = NULL;
 771     mSamplesMax = 0;
 772     mSamplesPos = 0;
 773     mSamplesLen = 0;
 774
 775     mDecodedFrame.reset(av_frame_alloc());
 776     if(!mDecodedFrame)
 777     {
 778         std::cerr<< "Failed to allocate audio frame" <<std::endl;
 779         goto finish;
 780     }
 781
 782     mSwresCtx.reset(swr_alloc_set_opts(nullptr,
 783         mDstChanLayout, mDstSampleFmt, mCodecCtx->sample_rate,
 784         mCodecCtx->channel_layout ? mCodecCtx->channel_layout :
 785             (uint64_t)av_get_default_channel_layout(mCodecCtx->channels),
 786         mCodecCtx->sample_fmt, mCodecCtx->sample_rate,
 787         0, nullptr
 788     ));
 789     if(!mSwresCtx || swr_init(mSwresCtx.get()) != 0)
 790     {
 791         std::cerr<< "Failed to initialize audio converter" <<std::endl;
 792         goto finish;
 793     }
 794
 795     mBuffers.assign(AudioBufferTotalTime / AudioBufferTime, 0);
 796     alGenBuffers(mBuffers.size(), mBuffers.data());
 797     alGenSources(1, &mSource);
 798
 799     if(EnableDirectOut)
 800         alSourcei(mSource, AL_DIRECT_CHANNELS_SOFT, AL_TRUE);
 801
 802     if(alGetError() != AL_NO_ERROR)
 803         goto finish;
 804
 805     if(!alBufferStorageSOFT)
 806         samples = av_malloc(buffer_len);
 807     else
 808     {
 809         for(ALuint bufid : mBuffers)
 810             alBufferStorageSOFT(bufid, mFormat, nullptr, buffer_len, mCodecCtx->sample_rate,
 811                                 AL_MAP_WRITE_BIT_SOFT);
 812         if(alGetError() != AL_NO_ERROR)
 813         {
 814             fprintf(stderr, "Failed to use mapped buffers\n");
 815             samples = av_malloc(buffer_len);
 816         }
 817     }
 818
 819     while(alGetError() == AL_NO_ERROR && !mMovie.mQuit.load(std::memory_order_relaxed))
 820     {
 821         /* First remove any processed buffers. */
 822         ALint processed;
 823         alGetSourcei(mSource, AL_BUFFERS_PROCESSED, &processed);
 824         while(processed > 0)
 825         {
 826             std::array<ALuint,4> bids;
 827             alSourceUnqueueBuffers(mSource, std::min<ALsizei>(bids.size(), processed),
 828                                    bids.data());
 829             processed -= std::min<ALsizei>(bids.size(), processed);
 830         }
 831
 832         /* Refill the buffer queue. */
 833         ALint queued;
 834         alGetSourcei(mSource, AL_BUFFERS_QUEUED, &queued);
 835         while((ALuint)queued < mBuffers.size())
 836         {
 837             ALuint bufid = mBuffers[mBufferIdx];
 838
 839             uint8_t *ptr = reinterpret_cast<uint8_t*>(
 840                 samples ? samples : alMapBufferSOFT(bufid, 0, buffer_len, AL_MAP_WRITE_BIT_SOFT)
 841             );
 842             if(!ptr) break;
 843
 844             /* Read the next chunk of data, filling the buffer, and queue it on
 845              * the source */
 846             bool got_audio = readAudio(ptr, buffer_len);
 847             if(!samples) alUnmapBufferSOFT(bufid);
 848             if(!got_audio) break;
 849
 850             if(samples)
 851                 alBufferData(bufid, mFormat, samples, buffer_len, mCodecCtx->sample_rate);
 852
 853             alSourceQueueBuffers(mSource, 1, &bufid);
 854             mBufferIdx = (mBufferIdx+1) % mBuffers.size();
 855             ++queued;
 856         }
 857         if(queued == 0)
 858             break;
 859
 860         /* Check that the source is playing. */
 861         ALint state;
 862         alGetSourcei(mSource, AL_SOURCE_STATE, &state);
 863         if(state == AL_STOPPED)
 864         {
 865             /* AL_STOPPED means there was an underrun. Clear the buffer queue
 866              * since this likely means we're late, and rewind the source to get
 867              * it back into an AL_INITIAL state.
 868              */
 869             alSourceRewind(mSource);
 870             alSourcei(mSource, AL_BUFFER, 0);
 871             continue;
 872         }
 873
 874         /* (re)start the source if needed, and wait for a buffer to finish */
 875         if(state != AL_PLAYING && state != AL_PAUSED &&
 876            mMovie.mPlaying.load(std::memory_order_relaxed))
 877             startPlayback();
 878
 879         lock.unlock();
 880         SDL_Delay((AudioBufferTime/3).count());
 881         lock.lock();
 882     }
 883
 884     alSourceRewind(mSource);
 885     alSourcei(mSource, AL_BUFFER, 0);
 886
 887 finish:
 888     av_freep(&samples);
 889
 890     return 0;
 891 }
 892
 893
 894 nanoseconds VideoState::getClock()
 895 {
 896     /* NOTE: This returns incorrect times while not playing. */
 897     auto delta = get_avtime() - mCurrentPtsTime;
 898     return mCurrentPts + delta;
 899 }
 900
 901 bool VideoState::isBufferFilled()
 902 {
 903     std::unique_lock<std::mutex> lock(mPictQMutex);
 904     return mPictQSize >= mPictQ.size();
 905 }
 906
 907 Uint32 SDLCALL VideoState::sdl_refresh_timer_cb(Uint32 /*interval*/, void *opaque)
 908 {
 909     SDL_Event evt{};
 910     evt.user.type = FF_REFRESH_EVENT;
 911     evt.user.data1 = opaque;
 912     SDL_PushEvent(&evt);
 913     return 0; /* 0 means stop timer */
 914 }
 915
 916 /* Schedules an FF_REFRESH_EVENT event to occur in 'delay' ms. */
 917 void VideoState::schedRefresh(milliseconds delay)
 918 {
 919     SDL_AddTimer(delay.count(), sdl_refresh_timer_cb, this);
 920 }
 921
 922 /* Called by VideoState::refreshTimer to display the next video frame. */
 923 void VideoState::display(SDL_Window *screen, SDL_Renderer *renderer)
 924 {
 925     Picture *vp = &mPictQ[mPictQRead];
 926
 927     if(!vp->mImage)
 928         return;
 929
 930     float aspect_ratio;
 931     int win_w, win_h;
 932     int w, h, x, y;
 933
 934     if(mCodecCtx->sample_aspect_ratio.num == 0)
 935         aspect_ratio = 0.0f;
 936     else
 937     {
 938         aspect_ratio = av_q2d(mCodecCtx->sample_aspect_ratio) * mCodecCtx->width /
 939                        mCodecCtx->height;
 940     }
 941     if(aspect_ratio <= 0.0f)
 942         aspect_ratio = (float)mCodecCtx->width / (float)mCodecCtx->height;
 943
 944     SDL_GetWindowSize(screen, &win_w, &win_h);
 945     h = win_h;
 946     w = ((int)rint(h * aspect_ratio) + 3) & ~3;
 947     if(w > win_w)
 948     {
 949         w = win_w;
 950         h = ((int)rint(w / aspect_ratio) + 3) & ~3;
 951     }
 952     x = (win_w - w) / 2;
 953     y = (win_h - h) / 2;
 954
 955     SDL_Rect src_rect{ 0, 0, vp->mWidth, vp->mHeight };
 956     SDL_Rect dst_rect{ x, y, w, h };
 957     SDL_RenderCopy(renderer, vp->mImage, &src_rect, &dst_rect);
 958     SDL_RenderPresent(renderer);
 959 }
 960
 961 /* FF_REFRESH_EVENT handler called on the main thread where the SDL_Renderer
 962  * was created. It handles the display of the next decoded video frame (if not
 963  * falling behind), and sets up the timer for the following video frame.
 964  */
 965 void VideoState::refreshTimer(SDL_Window *screen, SDL_Renderer *renderer)
 966 {
 967     if(!mStream)
 968     {
 969         if(mEOS)
 970         {
 971             mFinalUpdate = true;
 972             std::unique_lock<std::mutex>(mPictQMutex).unlock();
 973             mPictQCond.notify_all();
 974             return;
 975         }
 976         schedRefresh(milliseconds(100));
 977         return;
 978     }
 979     if(!mMovie.mPlaying.load(std::memory_order_relaxed))
 980     {
 981         schedRefresh(milliseconds(1));
 982         return;
 983     }
 984
 985     std::unique_lock<std::mutex> lock(mPictQMutex);
 986 retry:
 987     if(mPictQSize == 0)
 988     {
 989         if(mEOS)
 990             mFinalUpdate = true;
 991         else
 992             schedRefresh(milliseconds(1));
 993         lock.unlock();
 994         mPictQCond.notify_all();
 995         return;
 996     }
 997
 998     Picture *vp = &mPictQ[mPictQRead];
 999     mCurrentPts = vp->mPts;
1000     mCurrentPtsTime = get_avtime();
1001
1002     /* Get delay using the frame pts and the pts from last frame. */
1003     auto delay = vp->mPts - mFrameLastPts;
1004     if(delay <= seconds::zero() || delay >= seconds(1))
1005     {
1006         /* If incorrect delay, use previous one. */
1007         delay = mFrameLastDelay;
1008     }
1009     /* Save for next frame. */
1010     mFrameLastDelay = delay;
1011     mFrameLastPts = vp->mPts;
1012
1013     /* Update delay to sync to clock if not master source. */
1014     if(mMovie.mAVSyncType != SyncMaster::Video)
1015     {
1016         auto ref_clock = mMovie.getMasterClock();
1017         auto diff = vp->mPts - ref_clock;
1018
1019         /* Skip or repeat the frame. Take delay into account. */
1020         auto sync_threshold = std::min<nanoseconds>(delay, VideoSyncThreshold);
1021         if(!(diff < AVNoSyncThreshold && diff > -AVNoSyncThreshold))
1022         {
1023             if(diff <= -sync_threshold)
1024                 delay = nanoseconds::zero();
1025             else if(diff >= sync_threshold)
1026                 delay *= 2;
1027         }
1028     }
1029
1030     mFrameTimer += delay;
1031     /* Compute the REAL delay. */
1032     auto actual_delay = mFrameTimer - get_avtime();
1033     if(!(actual_delay >= VideoSyncThreshold))
1034     {
1035         /* We don't have time to handle this picture, just skip to the next one. */
1036         mPictQRead = (mPictQRead+1)%mPictQ.size();
1037         mPictQSize--;
1038         goto retry;
1039     }
1040     schedRefresh(std::chrono::duration_cast<milliseconds>(actual_delay));
1041
1042     /* Show the picture! */
1043     display(screen, renderer);
1044
1045     /* Update queue for next picture. */
1046     mPictQRead = (mPictQRead+1)%mPictQ.size();
1047     mPictQSize--;
1048     lock.unlock();
1049     mPictQCond.notify_all();
1050 }
1051
1052 /* FF_UPDATE_EVENT handler, updates the picture's texture. It's called on the
1053  * main thread where the renderer was created.
1054  */
1055 void VideoState::updatePicture(SDL_Window *screen, SDL_Renderer *renderer)
1056 {
1057     Picture *vp = &mPictQ[mPictQWrite];
1058     bool fmt_updated = false;
1059
1060     /* allocate or resize the buffer! */
1061     if(!vp->mImage || vp->mWidth != mCodecCtx->width || vp->mHeight != mCodecCtx->height)
1062     {
1063         fmt_updated = true;
1064         if(vp->mImage)
1065             SDL_DestroyTexture(vp->mImage);
1066         vp->mImage = SDL_CreateTexture(
1067             renderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING,
1068             mCodecCtx->coded_width, mCodecCtx->coded_height
1069         );
1070         if(!vp->mImage)
1071             std::cerr<< "Failed to create YV12 texture!" <<std::endl;
1072         vp->mWidth = mCodecCtx->width;
1073         vp->mHeight = mCodecCtx->height;
1074
1075         if(mFirstUpdate && vp->mWidth > 0 && vp->mHeight > 0)
1076         {
1077             /* For the first update, set the window size to the video size. */
1078             mFirstUpdate = false;
1079
1080             int w = vp->mWidth;
1081             int h = vp->mHeight;
1082             if(mCodecCtx->sample_aspect_ratio.den != 0)
1083             {
1084                 double aspect_ratio = av_q2d(mCodecCtx->sample_aspect_ratio);
1085                 if(aspect_ratio >= 1.0)
1086                     w = (int)(w*aspect_ratio + 0.5);
1087                 else if(aspect_ratio > 0.0)
1088                     h = (int)(h/aspect_ratio + 0.5);
1089             }
1090             SDL_SetWindowSize(screen, w, h);
1091         }
1092     }
1093
1094     if(vp->mImage)
1095     {
1096         AVFrame *frame = mDecodedFrame.get();
1097         void *pixels = nullptr;
1098         int pitch = 0;
1099
1100         if(mCodecCtx->pix_fmt == AV_PIX_FMT_YUV420P)
1101             SDL_UpdateYUVTexture(vp->mImage, nullptr,
1102                 frame->data[0], frame->linesize[0],
1103                 frame->data[1], frame->linesize[1],
1104                 frame->data[2], frame->linesize[2]
1105             );
1106         else if(SDL_LockTexture(vp->mImage, nullptr, &pixels, &pitch) != 0)
1107             std::cerr<< "Failed to lock texture" <<std::endl;
1108         else
1109         {
1110             // Convert the image into YUV format that SDL uses
1111             int coded_w = mCodecCtx->coded_width;
1112             int coded_h = mCodecCtx->coded_height;
1113             int w = mCodecCtx->width;
1114             int h = mCodecCtx->height;
1115             if(!mSwscaleCtx || fmt_updated)
1116             {
1117                 mSwscaleCtx.reset(sws_getContext(
1118                     w, h, mCodecCtx->pix_fmt,
1119                     w, h, AV_PIX_FMT_YUV420P, 0,
1120                     nullptr, nullptr, nullptr
1121                 ));
1122             }
1123
1124             /* point pict at the queue */
1125             uint8_t *pict_data[3];
1126             pict_data[0] = reinterpret_cast<uint8_t*>(pixels);
1127             pict_data[1] = pict_data[0] + coded_w*coded_h;
1128             pict_data[2] = pict_data[1] + coded_w*coded_h/4;
1129
1130             int pict_linesize[3];
1131             pict_linesize[0] = pitch;
1132             pict_linesize[1] = pitch / 2;
1133             pict_linesize[2] = pitch / 2;
1134
1135             sws_scale(mSwscaleCtx.get(), (const uint8_t**)frame->data,
1136                       frame->linesize, 0, h, pict_data, pict_linesize);
1137             SDL_UnlockTexture(vp->mImage);
1138         }
1139     }
1140
1141     vp->mUpdated.store(true, std::memory_order_release);
1142     std::unique_lock<std::mutex>(mPictQMutex).unlock();
1143     mPictQCond.notify_one();
1144 }
1145
1146 int VideoState::queuePicture(nanoseconds pts)
1147 {
1148     /* Wait until we have space for a new pic */
1149     std::unique_lock<std::mutex> lock(mPictQMutex);
1150     while(mPictQSize >= mPictQ.size() && !mMovie.mQuit.load(std::memory_order_relaxed))
1151         mPictQCond.wait(lock);
1152     lock.unlock();
1153
1154     if(mMovie.mQuit.load(std::memory_order_relaxed))
1155         return -1;
1156
1157     Picture *vp = &mPictQ[mPictQWrite];
1158
1159     /* We have to create/update the picture in the main thread  */
1160     vp->mUpdated.store(false, std::memory_order_relaxed);
1161     SDL_Event evt{};
1162     evt.user.type = FF_UPDATE_EVENT;
1163     evt.user.data1 = this;
1164     SDL_PushEvent(&evt);
1165
1166     /* Wait until the picture is updated. */
1167     lock.lock();
1168     while(!vp->mUpdated.load(std::memory_order_relaxed))
1169     {
1170         if(mMovie.mQuit.load(std::memory_order_relaxed))
1171             return -1;
1172         mPictQCond.wait(lock);
1173     }
1174     if(mMovie.mQuit.load(std::memory_order_relaxed))
1175         return -1;
1176     vp->mPts = pts;
1177
1178     mPictQWrite = (mPictQWrite+1)%mPictQ.size();
1179     mPictQSize++;
1180     lock.unlock();
1181
1182     return 0;
1183 }
1184
1185 int VideoState::handler()
1186 {
1187     mDecodedFrame.reset(av_frame_alloc());
1188     while(!mMovie.mQuit.load(std::memory_order_relaxed))
1189     {
1190         std::unique_lock<std::mutex> lock(mQueueMtx);
1191         /* Decode video frame */
1192         int ret = avcodec_receive_frame(mCodecCtx.get(), mDecodedFrame.get());
1193         if(ret == AVERROR(EAGAIN))
1194         {
1195             mMovie.mSendDataGood.clear(std::memory_order_relaxed);
1196             std::unique_lock<std::mutex>(mMovie.mSendMtx).unlock();
1197             mMovie.mSendCond.notify_one();
1198             do {
1199                 mQueueCond.wait(lock);
1200                 ret = avcodec_receive_frame(mCodecCtx.get(), mDecodedFrame.get());
1201             } while(ret == AVERROR(EAGAIN));
1202         }
1203         lock.unlock();
1204         if(ret == AVERROR_EOF) break;
1205         mMovie.mSendDataGood.clear(std::memory_order_relaxed);
1206         mMovie.mSendCond.notify_one();
1207         if(ret < 0)
1208         {
1209             std::cerr<< "Failed to decode frame: "<<ret <<std::endl;
1210             continue;
1211         }
1212
1213         /* Get the PTS for this frame. */
1214         nanoseconds pts;
1215         if(mDecodedFrame->best_effort_timestamp != AV_NOPTS_VALUE)
1216             mClock = std::chrono::duration_cast<nanoseconds>(
1217                 seconds_d64(av_q2d(mStream->time_base)*mDecodedFrame->best_effort_timestamp)
1218             );
1219         pts = mClock;
1220
1221         /* Update the video clock to the next expected PTS. */
1222         auto frame_delay = av_q2d(mCodecCtx->time_base);
1223         frame_delay += mDecodedFrame->repeat_pict * (frame_delay * 0.5);
1224         mClock += std::chrono::duration_cast<nanoseconds>(seconds_d64(frame_delay));
1225
1226         if(queuePicture(pts) < 0)
1227             break;
1228         av_frame_unref(mDecodedFrame.get());
1229     }
1230     mEOS = true;
1231
1232     std::unique_lock<std::mutex> lock(mPictQMutex);
1233     if(mMovie.mQuit.load(std::memory_order_relaxed))
1234     {
1235         mPictQRead = 0;
1236         mPictQWrite = 0;
1237         mPictQSize = 0;
1238     }
1239     while(!mFinalUpdate)
1240         mPictQCond.wait(lock);
1241
1242     return 0;
1243 }
1244
1245
1246 int MovieState::decode_interrupt_cb(void *ctx)
1247 {
1248     return reinterpret_cast<MovieState*>(ctx)->mQuit.load(std::memory_order_relaxed);
1249 }
1250
1251 bool MovieState::prepare()
1252 {
1253     AVIOContext *avioctx = nullptr;
1254     AVIOInterruptCB intcb = { decode_interrupt_cb, this };
1255     if(avio_open2(&avioctx, mFilename.c_str(), AVIO_FLAG_READ, &intcb, nullptr))
1256     {
1257         std::cerr<< "Failed to open "<<mFilename <<std::endl;
1258         return false;
1259     }
1260     mIOContext.reset(avioctx);
1261
1262     /* Open movie file. If avformat_open_input fails it will automatically free
1263      * this context, so don't set it onto a smart pointer yet.
1264      */
1265     AVFormatContext *fmtctx = avformat_alloc_context();
1266     fmtctx->pb = mIOContext.get();
1267     fmtctx->interrupt_callback = intcb;
1268     if(avformat_open_input(&fmtctx, mFilename.c_str(), nullptr, nullptr) != 0)
1269     {
1270         std::cerr<< "Failed to open "<<mFilename <<std::endl;
1271         return false;
1272     }
1273     mFormatCtx.reset(fmtctx);
1274
1275     /* Retrieve stream information */
1276     if(avformat_find_stream_info(mFormatCtx.get(), nullptr) < 0)
1277     {
1278         std::cerr<< mFilename<<": failed to find stream info" <<std::endl;
1279         return false;
1280     }
1281
1282     mVideo.schedRefresh(milliseconds(40));
1283
1284     mParseThread = std::thread(std::mem_fn(&MovieState::parse_handler), this);
1285     return true;
1286 }
1287
1288 void MovieState::setTitle(SDL_Window *window)
1289 {
1290     auto pos1 = mFilename.rfind('/');
1291     auto pos2 = mFilename.rfind('\\');
1292     auto fpos = ((pos1 == std::string::npos) ? pos2 :
1293                  (pos2 == std::string::npos) ? pos1 :
1294                  std::max(pos1, pos2)) + 1;
1295     SDL_SetWindowTitle(window, (mFilename.substr(fpos)+" - "+AppName).c_str());
1296 }
1297
1298 nanoseconds MovieState::getClock()
1299 {
1300     if(!mPlaying.load(std::memory_order_relaxed))
1301         return nanoseconds::zero();
1302     return get_avtime() - mClockBase;
1303 }
1304
1305 nanoseconds MovieState::getMasterClock()
1306 {
1307     if(mAVSyncType == SyncMaster::Video)
1308         return mVideo.getClock();
1309     if(mAVSyncType == SyncMaster::Audio)
1310         return mAudio.getClock();
1311     return getClock();
1312 }
1313
1314 nanoseconds MovieState::getDuration()
1315 { return std::chrono::duration<int64_t,std::ratio<1,AV_TIME_BASE>>(mFormatCtx->duration); }
1316
1317 int MovieState::streamComponentOpen(int stream_index)
1318 {
1319     if(stream_index < 0 || (unsigned int)stream_index >= mFormatCtx->nb_streams)
1320         return -1;
1321
1322     /* Get a pointer to the codec context for the stream, and open the
1323      * associated codec.
1324      */
1325     AVCodecCtxPtr avctx(avcodec_alloc_context3(nullptr));
1326     if(!avctx) return -1;
1327
1328     if(avcodec_parameters_to_context(avctx.get(), mFormatCtx->streams[stream_index]->codecpar))
1329         return -1;
1330
1331     AVCodec *codec = avcodec_find_decoder(avctx->codec_id);
1332     if(!codec || avcodec_open2(avctx.get(), codec, nullptr) < 0)
1333     {
1334         std::cerr<< "Unsupported codec: "<<avcodec_get_name(avctx->codec_id)
1335                  << " (0x"<<std::hex<<avctx->codec_id<<std::dec<<")" <<std::endl;
1336         return -1;
1337     }
1338
1339     /* Initialize and start the media type handler */
1340     switch(avctx->codec_type)
1341     {
1342         case AVMEDIA_TYPE_AUDIO:
1343             mAudio.mStream = mFormatCtx->streams[stream_index];
1344             mAudio.mCodecCtx = std::move(avctx);
1345
1346             mAudioThread = std::thread(std::mem_fn(&AudioState::handler), &mAudio);
1347             break;
1348
1349         case AVMEDIA_TYPE_VIDEO:
1350             mVideo.mStream = mFormatCtx->streams[stream_index];
1351             mVideo.mCodecCtx = std::move(avctx);
1352
1353             mVideoThread = std::thread(std::mem_fn(&VideoState::handler), &mVideo);
1354             break;
1355
1356         default:
1357             return -1;
1358     }
1359
1360     return stream_index;
1361 }
1362
1363 int MovieState::parse_handler()
1364 {
1365     int video_index = -1;
1366     int audio_index = -1;
1367
1368     /* Dump information about file onto standard error */
1369     av_dump_format(mFormatCtx.get(), 0, mFilename.c_str(), 0);
1370
1371     /* Find the first video and audio streams */
1372     for(unsigned int i = 0;i < mFormatCtx->nb_streams;i++)
1373     {
1374         auto codecpar = mFormatCtx->streams[i]->codecpar;
1375         if(codecpar->codec_type == AVMEDIA_TYPE_VIDEO && video_index < 0)
1376             video_index = streamComponentOpen(i);
1377         else if(codecpar->codec_type == AVMEDIA_TYPE_AUDIO && audio_index < 0)
1378             audio_index = streamComponentOpen(i);
1379     }
1380
1381     if(video_index < 0 && audio_index < 0)
1382     {
1383         std::cerr<< mFilename<<": could not open codecs" <<std::endl;
1384         mQuit = true;
1385     }
1386
1387     PacketQueue audio_queue, video_queue;
1388     bool input_finished = false;
1389
1390     /* Main packet reading/dispatching loop */
1391     while(!mQuit.load(std::memory_order_relaxed) && !input_finished)
1392     {
1393         AVPacket packet;
1394         if(av_read_frame(mFormatCtx.get(), &packet) < 0)
1395             input_finished = true;
1396         else
1397         {
1398             /* Copy the packet into the queue it's meant for. */
1399             if(packet.stream_index == video_index)
1400                 video_queue.put(&packet);
1401             else if(packet.stream_index == audio_index)
1402                 audio_queue.put(&packet);
1403             av_packet_unref(&packet);
1404         }
1405
1406         do {
1407             /* Send whatever queued packets we have. */
1408             if(!audio_queue.empty())
1409             {
1410                 std::unique_lock<std::mutex> lock(mAudio.mQueueMtx);
1411                 int ret;
1412                 do {
1413                     ret = avcodec_send_packet(mAudio.mCodecCtx.get(), audio_queue.front());
1414                     if(ret != AVERROR(EAGAIN)) audio_queue.pop();
1415                 } while(ret != AVERROR(EAGAIN) && !audio_queue.empty());
1416                 lock.unlock();
1417                 mAudio.mQueueCond.notify_one();
1418             }
1419             if(!video_queue.empty())
1420             {
1421                 std::unique_lock<std::mutex> lock(mVideo.mQueueMtx);
1422                 int ret;
1423                 do {
1424                     ret = avcodec_send_packet(mVideo.mCodecCtx.get(), video_queue.front());
1425                     if(ret != AVERROR(EAGAIN)) video_queue.pop();
1426                 } while(ret != AVERROR(EAGAIN) && !video_queue.empty());
1427                 lock.unlock();
1428                 mVideo.mQueueCond.notify_one();
1429             }
1430             /* If the queues are completely empty, or it's not full and there's
1431              * more input to read, go get more.
1432              */
1433             size_t queue_size = audio_queue.totalSize() + video_queue.totalSize();
1434             if(queue_size == 0 || (queue_size < MAX_QUEUE_SIZE && !input_finished))
1435                 break;
1436
1437             if(!mPlaying.load(std::memory_order_relaxed))
1438             {
1439                 if((!mAudio.mCodecCtx || mAudio.isBufferFilled()) &&
1440                    (!mVideo.mCodecCtx || mVideo.isBufferFilled()))
1441                 {
1442                     /* Set the base time 50ms ahead of the current av time. */
1443                     mClockBase = get_avtime() + milliseconds(50);
1444                     mVideo.mCurrentPtsTime = mClockBase;
1445                     mVideo.mFrameTimer = mVideo.mCurrentPtsTime;
1446                     mAudio.startPlayback();
1447                     mPlaying.store(std::memory_order_release);
1448                 }
1449             }
1450             /* Nothing to send or get for now, wait a bit and try again. */
1451             { std::unique_lock<std::mutex> lock(mSendMtx);
1452                 if(mSendDataGood.test_and_set(std::memory_order_relaxed))
1453                     mSendCond.wait_for(lock, milliseconds(10));
1454             }
1455         } while(!mQuit.load(std::memory_order_relaxed));
1456     }
1457     /* Pass a null packet to finish the send buffers (the receive functions
1458      * will get AVERROR_EOF when emptied).
1459      */
1460     if(mVideo.mCodecCtx)
1461     {
1462         { std::lock_guard<std::mutex> lock(mVideo.mQueueMtx);
1463             avcodec_send_packet(mVideo.mCodecCtx.get(), nullptr);
1464         }
1465         mVideo.mQueueCond.notify_one();
1466     }
1467     if(mAudio.mCodecCtx)
1468     {
1469         { std::lock_guard<std::mutex> lock(mAudio.mQueueMtx);
1470             avcodec_send_packet(mAudio.mCodecCtx.get(), nullptr);
1471         }
1472         mAudio.mQueueCond.notify_one();
1473     }
1474     video_queue.clear();
1475     audio_queue.clear();
1476
1477     /* all done - wait for it */
1478     if(mVideoThread.joinable())
1479         mVideoThread.join();
1480     if(mAudioThread.joinable())
1481         mAudioThread.join();
1482
1483     mVideo.mEOS = true;
1484     std::unique_lock<std::mutex> lock(mVideo.mPictQMutex);
1485     while(!mVideo.mFinalUpdate)
1486         mVideo.mPictQCond.wait(lock);
1487     lock.unlock();
1488
1489     SDL_Event evt{};
1490     evt.user.type = FF_MOVIE_DONE_EVENT;
1491     SDL_PushEvent(&evt);
1492
1493     return 0;
1494 }
1495
1496
1497 // Helper class+method to print the time with human-readable formatting.
1498 struct PrettyTime {
1499     seconds mTime;
1500 };
1501 inline std::ostream &operator<<(std::ostream &os, const PrettyTime &rhs)
1502 {
1503     using hours = std::chrono::hours;
1504     using minutes = std::chrono::minutes;
1505     using std::chrono::duration_cast;
1506
1507     seconds t = rhs.mTime;
1508     if(t.count() < 0)
1509     {
1510         os << '-';
1511         t *= -1;
1512     }
1513
1514     // Only handle up to hour formatting
1515     if(t >= hours(1))
1516         os << duration_cast<hours>(t).count() << 'h' << std::setfill('0') << std::setw(2)
1517            << (duration_cast<minutes>(t).count() % 60) << 'm';
1518     else
1519         os << duration_cast<minutes>(t).count() << 'm' << std::setfill('0');
1520     os << std::setw(2) << (duration_cast<seconds>(t).count() % 60) << 's' << std::setw(0)
1521        << std::setfill(' ');
1522     return os;
1523 }
1524
1525 } // namespace
1526
1527
1528 int main(int argc, char *argv[])
1529 {
1530     std::unique_ptr<MovieState> movState;
1531
1532     if(argc < 2)
1533     {
1534         std::cerr<< "Usage: "<<argv[0]<<" [-device <device name>] [-direct] <files...>" <<std::endl;
1535         return 1;
1536     }
1537     /* Register all formats and codecs */
1538     av_register_all();
1539     /* Initialize networking protocols */
1540     avformat_network_init();
1541
1542     if(SDL_Init(SDL_INIT_VIDEO | SDL_INIT_TIMER))
1543     {
1544         std::cerr<< "Could not initialize SDL - <<"<<SDL_GetError() <<std::endl;
1545         return 1;
1546     }
1547
1548     /* Make a window to put our video */
1549     SDL_Window *screen = SDL_CreateWindow(AppName.c_str(), 0, 0, 640, 480, SDL_WINDOW_RESIZABLE);
1550     if(!screen)
1551     {
1552         std::cerr<< "SDL: could not set video mode - exiting" <<std::endl;
1553         return 1;
1554     }
1555     /* Make a renderer to handle the texture image surface and rendering. */
1556     Uint32 render_flags = SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC;
1557     SDL_Renderer *renderer = SDL_CreateRenderer(screen, -1, render_flags);
1558     if(renderer)
1559     {
1560         SDL_RendererInfo rinf{};
1561         bool ok = false;
1562
1563         /* Make sure the renderer supports IYUV textures. If not, fallback to a
1564          * software renderer. */
1565         if(SDL_GetRendererInfo(renderer, &rinf) == 0)
1566         {
1567             for(Uint32 i = 0;!ok && i < rinf.num_texture_formats;i++)
1568                 ok = (rinf.texture_formats[i] == SDL_PIXELFORMAT_IYUV);
1569         }
1570         if(!ok)
1571         {
1572             std::cerr<< "IYUV pixelformat textures not supported on renderer "<<rinf.name <<std::endl;
1573             SDL_DestroyRenderer(renderer);
1574             renderer = nullptr;
1575         }
1576     }
1577     if(!renderer)
1578     {
1579         render_flags = SDL_RENDERER_SOFTWARE | SDL_RENDERER_PRESENTVSYNC;
1580         renderer = SDL_CreateRenderer(screen, -1, render_flags);
1581     }
1582     if(!renderer)
1583     {
1584         std::cerr<< "SDL: could not create renderer - exiting" <<std::endl;
1585         return 1;
1586     }
1587     SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
1588     SDL_RenderFillRect(renderer, nullptr);
1589     SDL_RenderPresent(renderer);
1590
1591     /* Open an audio device */
1592     int fileidx = 1;
1593     ALCdevice *device = [argc,argv,&fileidx]() -> ALCdevice*
1594     {
1595         ALCdevice *dev = NULL;
1596         if(argc > 3 && strcmp(argv[1], "-device") == 0)
1597         {
1598             fileidx = 3;
1599             dev = alcOpenDevice(argv[2]);
1600             if(dev) return dev;
1601             std::cerr<< "Failed to open \""<<argv[2]<<"\" - trying default" <<std::endl;
1602         }
1603         return alcOpenDevice(nullptr);
1604     }();
1605     ALCcontext *context = alcCreateContext(device, nullptr);
1606     if(!context || alcMakeContextCurrent(context) == ALC_FALSE)
1607     {
1608         std::cerr<< "Failed to set up audio device" <<std::endl;
1609         if(context)
1610             alcDestroyContext(context);
1611         return 1;
1612     }
1613
1614     const ALCchar *name = nullptr;
1615     if(alcIsExtensionPresent(device, "ALC_ENUMERATE_ALL_EXT"))
1616         name = alcGetString(device, ALC_ALL_DEVICES_SPECIFIER);
1617     if(!name || alcGetError(device) != AL_NO_ERROR)
1618         name = alcGetString(device, ALC_DEVICE_SPECIFIER);
1619     std::cout<< "Opened \""<<name<<"\"" <<std::endl;
1620
1621     if(alcIsExtensionPresent(device, "ALC_SOFT_device_clock"))
1622     {
1623         std::cout<< "Found ALC_SOFT_device_clock" <<std::endl;
1624         alcGetInteger64vSOFT = reinterpret_cast<LPALCGETINTEGER64VSOFT>(
1625             alcGetProcAddress(device, "alcGetInteger64vSOFT")
1626         );
1627     }
1628
1629     if(alIsExtensionPresent("AL_SOFT_source_latency"))
1630     {
1631         std::cout<< "Found AL_SOFT_source_latency" <<std::endl;
1632         alGetSourcei64vSOFT = reinterpret_cast<LPALGETSOURCEI64VSOFT>(
1633             alGetProcAddress("alGetSourcei64vSOFT")
1634         );
1635     }
1636     if(alIsExtensionPresent("AL_SOFTX_map_buffer"))
1637     {
1638         std::cout<< "Found AL_SOFT_map_buffer" <<std::endl;
1639         alBufferStorageSOFT = reinterpret_cast<LPALBUFFERSTORAGESOFT>(
1640             alGetProcAddress("alBufferStorageSOFT"));
1641         alMapBufferSOFT = reinterpret_cast<LPALMAPBUFFERSOFT>(
1642             alGetProcAddress("alMapBufferSOFT"));
1643         alUnmapBufferSOFT = reinterpret_cast<LPALUNMAPBUFFERSOFT>(
1644             alGetProcAddress("alUnmapBufferSOFT"));
1645     }
1646
1647     if(fileidx < argc && strcmp(argv[fileidx], "-direct") == 0)
1648     {
1649         ++fileidx;
1650         if(!alIsExtensionPresent("AL_SOFT_direct_channels"))
1651             std::cerr<< "AL_SOFT_direct_channels not supported for direct output" <<std::endl;
1652         else
1653         {
1654             std::cout<< "Found AL_SOFT_direct_channels" <<std::endl;
1655             EnableDirectOut = true;
1656         }
1657     }
1658
1659     while(fileidx < argc && !movState)
1660     {
1661         movState = std::unique_ptr<MovieState>(new MovieState(argv[fileidx++]));
1662         if(!movState->prepare()) movState = nullptr;
1663     }
1664     if(!movState)
1665     {
1666         std::cerr<< "Could not start a video" <<std::endl;
1667         return 1;
1668     }
1669     movState->setTitle(screen);
1670
1671     /* Default to going to the next movie at the end of one. */
1672     enum class EomAction {
1673         Next, Quit
1674     } eom_action = EomAction::Next;
1675     seconds last_time(-1);
1676     SDL_Event event;
1677     while(1)
1678     {
1679         int have_evt = SDL_WaitEventTimeout(&event, 10);
1680
1681         auto cur_time = std::chrono::duration_cast<seconds>(movState->getMasterClock());
1682         if(cur_time != last_time)
1683         {
1684             auto end_time = std::chrono::duration_cast<seconds>(movState->getDuration());
1685             std::cout<< "\r "<<PrettyTime{cur_time}<<" / "<<PrettyTime{end_time} <<std::flush;
1686             last_time = cur_time;
1687         }
1688         if(!have_evt) continue;
1689
1690         switch(event.type)
1691         {
1692             case SDL_KEYDOWN:
1693                 switch(event.key.keysym.sym)
1694                 {
1695                     case SDLK_ESCAPE:
1696                         movState->mQuit = true;
1697                         eom_action = EomAction::Quit;
1698                         break;
1699
1700                     case SDLK_n:
1701                         movState->mQuit = true;
1702                         eom_action = EomAction::Next;
1703                         break;
1704
1705                     default:
1706                         break;
1707                 }
1708                 break;
1709
1710             case SDL_WINDOWEVENT:
1711                 switch(event.window.event)
1712                 {
1713                     case SDL_WINDOWEVENT_RESIZED:
1714                         SDL_SetRenderDrawColor(renderer, 0, 0, 0, 255);
1715                         SDL_RenderFillRect(renderer, nullptr);
1716                         break;
1717
1718                     default:
1719                         break;
1720                 }
1721                 break;
1722
1723             case SDL_QUIT:
1724                 movState->mQuit = true;
1725                 eom_action = EomAction::Quit;
1726                 break;
1727
1728             case FF_UPDATE_EVENT:
1729                 reinterpret_cast<VideoState*>(event.user.data1)->updatePicture(
1730                     screen, renderer
1731                 );
1732                 break;
1733
1734             case FF_REFRESH_EVENT:
1735                 reinterpret_cast<VideoState*>(event.user.data1)->refreshTimer(
1736                     screen, renderer
1737                 );
1738                 break;
1739
1740             case FF_MOVIE_DONE_EVENT:
1741                 std::cout<<'\n';
1742                 last_time = seconds(-1);
1743                 if(eom_action != EomAction::Quit)
1744                 {
1745                     movState = nullptr;
1746                     while(fileidx < argc && !movState)
1747                     {
1748                         movState = std::unique_ptr<MovieState>(new MovieState(argv[fileidx++]));
1749                         if(!movState->prepare()) movState = nullptr;
1750                     }
1751                     if(movState)
1752                     {
1753                         movState->setTitle(screen);
1754                         break;
1755                     }
1756                 }
1757
1758                 /* Nothing more to play. Shut everything down and quit. */
1759                 movState = nullptr;
1760
1761                 alcMakeContextCurrent(nullptr);
1762                 alcDestroyContext(context);
1763                 alcCloseDevice(device);
1764
1765                 SDL_DestroyRenderer(renderer);
1766                 renderer = nullptr;
1767                 SDL_DestroyWindow(screen);
1768                 screen = nullptr;
1769
1770                 SDL_Quit();
1771                 exit(0);
1772
1773             default:
1774                 break;
1775         }
1776     }
1777
1778     std::cerr<< "SDL_WaitEvent error - "<<SDL_GetError() <<std::endl;
1779     return 1;
1780 }