media/ffmpeg/ffmpeg_regression_tests.cc

   1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4 //
   5 // Regression tests for FFmpeg.  Test files can be found in the internal media
   6 // test data directory:
   7 //
   8 //    svn://svn.chromium.org/chrome-internal/trunk/data/media/
   9 //
  10 // Simply add the custom_dep below to your gclient and sync:
  11 //
  12 //    "src/media/test/data/internal":
  13 //        "svn://svn.chromium.org/chrome-internal/trunk/data/media"
  14 //
  15 // Many of the files here do not cause issues outside of tooling, so you'll need
  16 // to run this test under ASAN, TSAN, and Valgrind to ensure that all issues are
  17 // caught.
  18 //
  19 // Test cases labeled FLAKY may not always pass, but they should never crash or
  20 // cause any kind of warnings or errors under tooling.
  21
  22 #include <string>
  23
  24 #include "base/bind.h"
  25 #include "media/test/pipeline_integration_test_base.h"
  26
  27 namespace media {
  28
  29 const char kRegressionTestDataPathPrefix[] = "internal/";
  30
  31 struct RegressionTestData {
  32   RegressionTestData(const char* filename,
  33                      PipelineStatus init_status,
  34                      PipelineStatus end_status)
  35       : filename(std::string(kRegressionTestDataPathPrefix) + filename),
  36         init_status(init_status),
  37         end_status(end_status) {}
  38
  39   std::string filename;
  40   PipelineStatus init_status;
  41   PipelineStatus end_status;
  42 };
  43
  44 // Used for tests which just need to run without crashing or tooling errors, but
  45 // which may have undefined PipelineStatus results.
  46 struct FlakyRegressionTestData {
  47   FlakyRegressionTestData(const char* filename)
  48       : filename(std::string(kRegressionTestDataPathPrefix) + filename) {
  49   }
  50
  51   std::string filename;
  52 };
  53
  54 class FFmpegRegressionTest
  55     : public testing::TestWithParam<RegressionTestData>,
  56       public PipelineIntegrationTestBase {
  57 };
  58
  59 class FlakyFFmpegRegressionTest
  60     : public testing::TestWithParam<FlakyRegressionTestData>,
  61       public PipelineIntegrationTestBase {
  62 };
  63
  64 #define FFMPEG_TEST_CASE(name, fn, init_status, end_status) \
  65   INSTANTIATE_TEST_CASE_P(                                  \
  66       name, FFmpegRegressionTest,                           \
  67       testing::Values(RegressionTestData(fn, init_status, end_status)));
  68
  69 #define FLAKY_FFMPEG_TEST_CASE(name, fn) \
  70     INSTANTIATE_TEST_CASE_P(FLAKY_##name, FlakyFFmpegRegressionTest, \
  71                             testing::Values(FlakyRegressionTestData(fn)));
  72
  73 // Test cases from issues.
  74 FFMPEG_TEST_CASE(Cr47325, "security/47325.mp4", PIPELINE_OK, PIPELINE_OK);
  75 FFMPEG_TEST_CASE(Cr47761, "crbug47761.ogg", PIPELINE_OK, PIPELINE_OK);
  76 FFMPEG_TEST_CASE(Cr50045, "crbug50045.mp4", PIPELINE_OK, PIPELINE_OK);
  77 FFMPEG_TEST_CASE(Cr62127, "crbug62127.webm", PIPELINE_OK, PIPELINE_OK);
  78 FFMPEG_TEST_CASE(Cr93620, "security/93620.ogg", PIPELINE_OK, PIPELINE_OK);
  79 FFMPEG_TEST_CASE(Cr100492,
  80                  "security/100492.webm",
  81                  DECODER_ERROR_NOT_SUPPORTED,
  82                  DECODER_ERROR_NOT_SUPPORTED);
  83 FFMPEG_TEST_CASE(Cr100543, "security/100543.webm", PIPELINE_OK, PIPELINE_OK);
  84 FFMPEG_TEST_CASE(Cr101458, "security/101458.webm", PIPELINE_OK, PIPELINE_OK);
  85 FFMPEG_TEST_CASE(Cr108416, "security/108416.webm", PIPELINE_OK, PIPELINE_OK);
  86 FFMPEG_TEST_CASE(Cr110849,
  87                  "security/110849.mkv",
  88                  DEMUXER_ERROR_COULD_NOT_OPEN,
  89                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
  90 FFMPEG_TEST_CASE(Cr112384, "security/112384.webm", PIPELINE_OK, PIPELINE_OK);
  91 FFMPEG_TEST_CASE(Cr112976, "security/112976.ogg", PIPELINE_OK, PIPELINE_OK);
  92 FFMPEG_TEST_CASE(Cr116927,
  93                  "security/116927.ogv",
  94                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS,
  95                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
  96 FFMPEG_TEST_CASE(Cr117912,
  97                  "security/117912.webm",
  98                  DEMUXER_ERROR_COULD_NOT_OPEN,
  99                  DEMUXER_ERROR_COULD_NOT_OPEN);
 100 FFMPEG_TEST_CASE(Cr123481, "security/123481.ogv", PIPELINE_OK, PIPELINE_OK);
 101 FFMPEG_TEST_CASE(Cr132779,
 102                  "security/132779.webm",
 103                  DEMUXER_ERROR_COULD_NOT_PARSE,
 104                  DEMUXER_ERROR_COULD_NOT_PARSE);
 105 FFMPEG_TEST_CASE(Cr140165, "security/140165.ogg", PIPELINE_OK, PIPELINE_OK);
 106 FFMPEG_TEST_CASE(Cr140647,
 107                  "security/140647.ogv",
 108                  DEMUXER_ERROR_COULD_NOT_OPEN,
 109                  DEMUXER_ERROR_COULD_NOT_OPEN);
 110 FFMPEG_TEST_CASE(Cr142738, "crbug142738.ogg", PIPELINE_OK, PIPELINE_OK);
 111 FFMPEG_TEST_CASE(Cr152691,
 112                  "security/152691.mp3",
 113                  PIPELINE_OK,
 114                  PIPELINE_ERROR_DECODE);
 115 FFMPEG_TEST_CASE(Cr161639, "security/161639.m4a", PIPELINE_OK, PIPELINE_OK);
 116 FFMPEG_TEST_CASE(Cr222754,
 117                  "security/222754.mp4",
 118                  PIPELINE_OK,
 119                  PIPELINE_ERROR_DECODE);
 120 FFMPEG_TEST_CASE(Cr234630a, "security/234630a.mov", PIPELINE_OK, PIPELINE_OK);
 121 FFMPEG_TEST_CASE(Cr234630b,
 122                  "security/234630b.mov",
 123                  PIPELINE_OK,
 124                  PIPELINE_ERROR_DECODE);
 125 FFMPEG_TEST_CASE(Cr242786, "security/242786.webm", PIPELINE_OK, PIPELINE_OK);
 126 // Test for out-of-bounds access with slightly corrupt file (detection logic
 127 // thinks it's a MONO file, but actually contains STEREO audio).
 128 FFMPEG_TEST_CASE(Cr275590, "security/275590.m4a", PIPELINE_OK, PIPELINE_OK);
 129 FFMPEG_TEST_CASE(Cr444522, "security/444522.mp4", PIPELINE_OK, PIPELINE_OK);
 130 FFMPEG_TEST_CASE(Cr444539,
 131                  "security/444539.m4a",
 132                  DEMUXER_ERROR_COULD_NOT_OPEN,
 133                  DEMUXER_ERROR_COULD_NOT_OPEN);
 134 FFMPEG_TEST_CASE(Cr444546,
 135                  "security/444546.mp4",
 136                  DEMUXER_ERROR_COULD_NOT_OPEN,
 137                  DEMUXER_ERROR_COULD_NOT_OPEN);
 138 FFMPEG_TEST_CASE(Cr449958,
 139                  "security/449958.webm",
 140                  PIPELINE_OK,
 141                  PIPELINE_ERROR_DECODE);
 142
 143 // General MP4 test cases.
 144 FFMPEG_TEST_CASE(MP4_0,
 145                  "security/aac.10419.mp4",
 146                  DEMUXER_ERROR_COULD_NOT_OPEN,
 147                  DEMUXER_ERROR_COULD_NOT_OPEN);
 148 FFMPEG_TEST_CASE(MP4_1,
 149                  "security/clockh264aac_200021889.mp4",
 150                  DEMUXER_ERROR_COULD_NOT_OPEN,
 151                  DEMUXER_ERROR_COULD_NOT_OPEN);
 152 FFMPEG_TEST_CASE(MP4_2,
 153                  "security/clockh264aac_200701257.mp4",
 154                  PIPELINE_OK,
 155                  PIPELINE_OK);
 156 FFMPEG_TEST_CASE(MP4_5,
 157                  "security/clockh264aac_3022500.mp4",
 158                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS,
 159                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
 160 FFMPEG_TEST_CASE(MP4_6,
 161                  "security/clockh264aac_344289.mp4",
 162                  PIPELINE_OK,
 163                  PIPELINE_OK);
 164 FFMPEG_TEST_CASE(MP4_7,
 165                  "security/clockh264mp3_187697.mp4",
 166                  PIPELINE_OK,
 167                  PIPELINE_OK);
 168 FFMPEG_TEST_CASE(MP4_8,
 169                  "security/h264.705767.mp4",
 170                  DEMUXER_ERROR_COULD_NOT_PARSE,
 171                  DEMUXER_ERROR_COULD_NOT_PARSE);
 172 FFMPEG_TEST_CASE(MP4_9,
 173                  "security/smclockmp4aac_1_0.mp4",
 174                  DEMUXER_ERROR_COULD_NOT_OPEN,
 175                  DEMUXER_ERROR_COULD_NOT_OPEN);
 176 FFMPEG_TEST_CASE(MP4_11, "security/null1.mp4", PIPELINE_OK, PIPELINE_OK);
 177 FFMPEG_TEST_CASE(MP4_16,
 178                  "security/looping2.mov",
 179                  DEMUXER_ERROR_COULD_NOT_OPEN,
 180                  DEMUXER_ERROR_COULD_NOT_OPEN);
 181 FFMPEG_TEST_CASE(MP4_17, "security/assert2.mov", PIPELINE_OK, PIPELINE_OK);
 182
 183 // General OGV test cases.
 184 FFMPEG_TEST_CASE(OGV_1,
 185                  "security/out.163.ogv",
 186                  DECODER_ERROR_NOT_SUPPORTED,
 187                  DECODER_ERROR_NOT_SUPPORTED);
 188 FFMPEG_TEST_CASE(OGV_2,
 189                  "security/out.391.ogv",
 190                  DECODER_ERROR_NOT_SUPPORTED,
 191                  DECODER_ERROR_NOT_SUPPORTED);
 192 FFMPEG_TEST_CASE(OGV_5,
 193                  "security/smclocktheora_1_0.ogv",
 194                  DECODER_ERROR_NOT_SUPPORTED,
 195                  DECODER_ERROR_NOT_SUPPORTED);
 196 FFMPEG_TEST_CASE(OGV_7,
 197                  "security/smclocktheora_1_102.ogv",
 198                  DECODER_ERROR_NOT_SUPPORTED,
 199                  DECODER_ERROR_NOT_SUPPORTED);
 200 FFMPEG_TEST_CASE(OGV_8,
 201                  "security/smclocktheora_1_104.ogv",
 202                  DECODER_ERROR_NOT_SUPPORTED,
 203                  DECODER_ERROR_NOT_SUPPORTED);
 204 FFMPEG_TEST_CASE(OGV_9,
 205                  "security/smclocktheora_1_110.ogv",
 206                  DECODER_ERROR_NOT_SUPPORTED,
 207                  DECODER_ERROR_NOT_SUPPORTED);
 208 FFMPEG_TEST_CASE(OGV_10,
 209                  "security/smclocktheora_1_179.ogv",
 210                  DECODER_ERROR_NOT_SUPPORTED,
 211                  DECODER_ERROR_NOT_SUPPORTED);
 212 FFMPEG_TEST_CASE(OGV_11,
 213                  "security/smclocktheora_1_20.ogv",
 214                  DECODER_ERROR_NOT_SUPPORTED,
 215                  DECODER_ERROR_NOT_SUPPORTED);
 216 FFMPEG_TEST_CASE(OGV_12,
 217                  "security/smclocktheora_1_723.ogv",
 218                  DECODER_ERROR_NOT_SUPPORTED,
 219                  DECODER_ERROR_NOT_SUPPORTED);
 220 FFMPEG_TEST_CASE(OGV_14,
 221                  "security/smclocktheora_2_10405.ogv",
 222                  PIPELINE_OK,
 223                  PIPELINE_OK);
 224 FFMPEG_TEST_CASE(OGV_15,
 225                  "security/smclocktheora_2_10619.ogv",
 226                  DECODER_ERROR_NOT_SUPPORTED,
 227                  DECODER_ERROR_NOT_SUPPORTED);
 228 FFMPEG_TEST_CASE(OGV_16,
 229                  "security/smclocktheora_2_1075.ogv",
 230                  DECODER_ERROR_NOT_SUPPORTED,
 231                  DECODER_ERROR_NOT_SUPPORTED);
 232 FFMPEG_TEST_CASE(OGV_17,
 233                  "security/vorbis.482086.ogv",
 234                  PIPELINE_OK,
 235                  PIPELINE_OK);
 236 FFMPEG_TEST_CASE(OGV_18,
 237                  "security/wav.711.ogv",
 238                  DECODER_ERROR_NOT_SUPPORTED,
 239                  DECODER_ERROR_NOT_SUPPORTED);
 240 FFMPEG_TEST_CASE(OGV_19,
 241                  "security/null1.ogv",
 242                  DECODER_ERROR_NOT_SUPPORTED,
 243                  DECODER_ERROR_NOT_SUPPORTED);
 244 FFMPEG_TEST_CASE(OGV_20,
 245                  "security/null2.ogv",
 246                  DECODER_ERROR_NOT_SUPPORTED,
 247                  DECODER_ERROR_NOT_SUPPORTED);
 248 FFMPEG_TEST_CASE(OGV_21,
 249                  "security/assert1.ogv",
 250                  DECODER_ERROR_NOT_SUPPORTED,
 251                  DECODER_ERROR_NOT_SUPPORTED);
 252 FFMPEG_TEST_CASE(OGV_22,
 253                  "security/assert2.ogv",
 254                  DECODER_ERROR_NOT_SUPPORTED,
 255                  DECODER_ERROR_NOT_SUPPORTED);
 256 FFMPEG_TEST_CASE(OGV_23,
 257                  "security/assert2.ogv",
 258                  DECODER_ERROR_NOT_SUPPORTED,
 259                  DECODER_ERROR_NOT_SUPPORTED);
 260
 261 // General WebM test cases.
 262 FFMPEG_TEST_CASE(WEBM_0, "security/memcpy.webm", PIPELINE_OK, PIPELINE_OK);
 263 FFMPEG_TEST_CASE(WEBM_1, "security/no-bug.webm", PIPELINE_OK, PIPELINE_OK);
 264 FFMPEG_TEST_CASE(WEBM_2,
 265                  "security/uninitialize.webm",
 266                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS,
 267                  DEMUXER_ERROR_NO_SUPPORTED_STREAMS);
 268 FFMPEG_TEST_CASE(WEBM_4,
 269                  "security/out.webm.68798.1929",
 270                  DECODER_ERROR_NOT_SUPPORTED,
 271                  DECODER_ERROR_NOT_SUPPORTED);
 272 FFMPEG_TEST_CASE(WEBM_5, "frame_size_change.webm", PIPELINE_OK, PIPELINE_OK);
 273
 274 // General MKV test cases.
 275 FFMPEG_TEST_CASE(MKV_0,
 276                  "security/nested_tags_lang.mka.627.628",
 277                  PIPELINE_OK,
 278                  PIPELINE_ERROR_DECODE);
 279 FFMPEG_TEST_CASE(MKV_1,
 280                  "security/nested_tags_lang.mka.667.628",
 281                  PIPELINE_OK,
 282                  PIPELINE_ERROR_DECODE);
 283
 284 // Allocate gigabytes of memory, likely can't be run on 32bit machines.
 285 FFMPEG_TEST_CASE(BIG_MEM_1,
 286                  "security/bigmem1.mov",
 287                  DEMUXER_ERROR_COULD_NOT_OPEN,
 288                  DEMUXER_ERROR_COULD_NOT_OPEN);
 289 FFMPEG_TEST_CASE(BIG_MEM_2,
 290                  "security/looping1.mov",
 291                  DEMUXER_ERROR_COULD_NOT_OPEN,
 292                  DEMUXER_ERROR_COULD_NOT_OPEN);
 293 FFMPEG_TEST_CASE(BIG_MEM_5,
 294                  "security/looping5.mov",
 295                  DEMUXER_ERROR_COULD_NOT_OPEN,
 296                  DEMUXER_ERROR_COULD_NOT_OPEN);
 297 FLAKY_FFMPEG_TEST_CASE(BIG_MEM_3, "security/looping3.mov");
 298 FLAKY_FFMPEG_TEST_CASE(BIG_MEM_4, "security/looping4.mov");
 299
 300 // Flaky under threading or for other reasons.  Per rbultje, most of these will
 301 // never be reliable since FFmpeg does not guarantee consistency in error cases.
 302 // We only really care that these don't cause crashes or errors under tooling.
 303 FLAKY_FFMPEG_TEST_CASE(Cr99652, "security/99652.webm");
 304 FLAKY_FFMPEG_TEST_CASE(Cr100464, "security/100464.webm");
 305 FLAKY_FFMPEG_TEST_CASE(Cr111342, "security/111342.ogm");
 306 FLAKY_FFMPEG_TEST_CASE(Cr368980, "security/368980.mp4");
 307 FLAKY_FFMPEG_TEST_CASE(OGV_0, "security/big_dims.ogv");
 308 FLAKY_FFMPEG_TEST_CASE(OGV_3, "security/smclock_1_0.ogv");
 309 FLAKY_FFMPEG_TEST_CASE(OGV_4, "security/smclock.ogv.1.0.ogv");
 310 FLAKY_FFMPEG_TEST_CASE(OGV_6, "security/smclocktheora_1_10000.ogv");
 311 FLAKY_FFMPEG_TEST_CASE(OGV_13, "security/smclocktheora_1_790.ogv");
 312 FLAKY_FFMPEG_TEST_CASE(MP4_3, "security/clockh264aac_300413969.mp4");
 313 FLAKY_FFMPEG_TEST_CASE(MP4_4, "security/clockh264aac_301350139.mp4");
 314 FLAKY_FFMPEG_TEST_CASE(MP4_12, "security/assert1.mov");
 315 FLAKY_FFMPEG_TEST_CASE(WEBM_3, "security/out.webm.139771.2965");
 316
 317 // Not really flaky, but can't pass the seek test.
 318 FLAKY_FFMPEG_TEST_CASE(MP4_10, "security/null1.m4a");
 319 FLAKY_FFMPEG_TEST_CASE(Cr112670, "security/112670.mp4");
 320
 321 TEST_P(FFmpegRegressionTest, BasicPlayback) {
 322   if (GetParam().init_status == PIPELINE_OK) {
 323     ASSERT_EQ(PIPELINE_OK, Start(GetParam().filename, kClockless));
 324     Play();
 325     ASSERT_EQ(GetParam().end_status, WaitUntilEndedOrError());
 326
 327     // Check for ended if the pipeline is expected to finish okay.
 328     if (GetParam().end_status == PIPELINE_OK) {
 329       ASSERT_TRUE(ended_);
 330
 331       // Tack a seek on the end to catch any seeking issues.
 332       Seek(base::TimeDelta::FromMilliseconds(0));
 333     }
 334   } else {
 335     // Don't bother checking the exact status as we only care that the
 336     // pipeline failed to start.
 337     EXPECT_NE(PIPELINE_OK, Start(GetParam().filename));
 338   }
 339 }
 340
 341 TEST_P(FlakyFFmpegRegressionTest, BasicPlayback) {
 342   if (Start(GetParam().filename, kClockless) == PIPELINE_OK) {
 343     Play();
 344     WaitUntilEndedOrError();
 345   }
 346 }
 347
 348 }  // namespace media