dom/media/DOMMediaStream.cpp

   1 /* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*-*/
   2 /* This Source Code Form is subject to the terms of the Mozilla Public
   3  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
   4  * You can obtain one at http://mozilla.org/MPL/2.0/. */
   5
   6 #include "DOMMediaStream.h"
   7
   8 #include "AudioCaptureTrack.h"
   9 #include "AudioChannelAgent.h"
  10 #include "AudioStreamTrack.h"
  11 #include "MediaTrackGraph.h"
  12 #include "MediaTrackGraphImpl.h"
  13 #include "MediaTrackListener.h"
  14 #include "Tracing.h"
  15 #include "VideoStreamTrack.h"
  16 #include "mozilla/dom/AudioTrack.h"
  17 #include "mozilla/dom/AudioTrackList.h"
  18 #include "mozilla/dom/DocGroup.h"
  19 #include "mozilla/dom/HTMLCanvasElement.h"
  20 #include "mozilla/dom/MediaStreamBinding.h"
  21 #include "mozilla/dom/MediaStreamTrackEvent.h"
  22 #include "mozilla/dom/Promise.h"
  23 #include "mozilla/dom/VideoTrack.h"
  24 #include "mozilla/dom/VideoTrackList.h"
  25 #include "mozilla/media/MediaUtils.h"
  26 #include "nsContentUtils.h"
  27 #include "nsGlobalWindowInner.h"
  28 #include "nsIUUIDGenerator.h"
  29 #include "nsPIDOMWindow.h"
  30 #include "nsProxyRelease.h"
  31 #include "nsRFPService.h"
  32 #include "nsServiceManagerUtils.h"
  33
  34 #ifdef LOG
  35 #  undef LOG
  36 #endif
  37
  38 using namespace mozilla;
  39 using namespace mozilla::dom;
  40 using namespace mozilla::layers;
  41 using namespace mozilla::media;
  42
  43 static LazyLogModule gMediaStreamLog("MediaStream");
  44 #define LOG(type, msg) MOZ_LOG(gMediaStreamLog, type, msg)
  45
  46 static bool ContainsLiveTracks(
  47     const nsTArray<RefPtr<MediaStreamTrack>>& aTracks) {
  48   for (const auto& track : aTracks) {
  49     if (track->ReadyState() == MediaStreamTrackState::Live) {
  50       return true;
  51     }
  52   }
  53
  54   return false;
  55 }
  56
  57 static bool ContainsLiveAudioTracks(
  58     const nsTArray<RefPtr<MediaStreamTrack>>& aTracks) {
  59   for (const auto& track : aTracks) {
  60     if (track->AsAudioStreamTrack() &&
  61         track->ReadyState() == MediaStreamTrackState::Live) {
  62       return true;
  63     }
  64   }
  65
  66   return false;
  67 }
  68
  69 class DOMMediaStream::PlaybackTrackListener : public MediaStreamTrackConsumer {
  70  public:
  71   NS_INLINE_DECL_REFCOUNTING(PlaybackTrackListener)
  72
  73   explicit PlaybackTrackListener(DOMMediaStream* aStream) : mStream(aStream) {}
  74
  75   void NotifyEnded(MediaStreamTrack* aTrack) override {
  76     if (!mStream) {
  77       return;
  78     }
  79
  80     if (!aTrack) {
  81       MOZ_ASSERT(false);
  82       return;
  83     }
  84
  85     MOZ_ASSERT(mStream->HasTrack(*aTrack));
  86     mStream->NotifyTrackRemoved(aTrack);
  87   }
  88
  89  protected:
  90   virtual ~PlaybackTrackListener() = default;
  91
  92   WeakPtr<DOMMediaStream> mStream;
  93 };
  94
  95 NS_IMPL_CYCLE_COLLECTION_CLASS(DOMMediaStream)
  96
  97 NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(DOMMediaStream,
  98                                                 DOMEventTargetHelper)
  99   tmp->Destroy();
 100   NS_IMPL_CYCLE_COLLECTION_UNLINK(mTracks)
 101   NS_IMPL_CYCLE_COLLECTION_UNLINK(mConsumersToKeepAlive)
 102   NS_IMPL_CYCLE_COLLECTION_UNLINK_WEAK_PTR
 103 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
 104
 105 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(DOMMediaStream,
 106                                                   DOMEventTargetHelper)
 107   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mTracks)
 108   NS_IMPL_CYCLE_COLLECTION_TRAVERSE(mConsumersToKeepAlive)
 109 NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
 110
 111 NS_IMPL_ADDREF_INHERITED(DOMMediaStream, DOMEventTargetHelper)
 112 NS_IMPL_RELEASE_INHERITED(DOMMediaStream, DOMEventTargetHelper)
 113
 114 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(DOMMediaStream)
 115   NS_INTERFACE_MAP_ENTRY_CONCRETE(DOMMediaStream)
 116 NS_INTERFACE_MAP_END_INHERITING(DOMEventTargetHelper)
 117
 118 DOMMediaStream::DOMMediaStream(nsPIDOMWindowInner* aWindow)
 119     : DOMEventTargetHelper(aWindow),
 120       mPlaybackTrackListener(MakeAndAddRef<PlaybackTrackListener>(this)) {
 121   nsresult rv;
 122   nsCOMPtr<nsIUUIDGenerator> uuidgen =
 123       do_GetService("@mozilla.org/uuid-generator;1", &rv);
 124
 125   if (NS_SUCCEEDED(rv) && uuidgen) {
 126     nsID uuid;
 127     memset(&uuid, 0, sizeof(uuid));
 128     rv = uuidgen->GenerateUUIDInPlace(&uuid);
 129     if (NS_SUCCEEDED(rv)) {
 130       char buffer[NSID_LENGTH];
 131       uuid.ToProvidedString(buffer);
 132       mID = NS_ConvertASCIItoUTF16(buffer);
 133     }
 134   }
 135 }
 136
 137 DOMMediaStream::~DOMMediaStream() { Destroy(); }
 138
 139 void DOMMediaStream::Destroy() {
 140   LOG(LogLevel::Debug, ("DOMMediaStream %p Being destroyed.", this));
 141   for (const auto& track : mTracks) {
 142     // We must remove ourselves from each track's principal change observer list
 143     // before we die.
 144     if (!track->Ended()) {
 145       track->RemoveConsumer(mPlaybackTrackListener);
 146     }
 147   }
 148   mTrackListeners.Clear();
 149 }
 150
 151 JSObject* DOMMediaStream::WrapObject(JSContext* aCx,
 152                                      JS::Handle<JSObject*> aGivenProto) {
 153   return dom::MediaStream_Binding::Wrap(aCx, this, aGivenProto);
 154 }
 155
 156 /* static */
 157 already_AddRefed<DOMMediaStream> DOMMediaStream::Constructor(
 158     const GlobalObject& aGlobal, ErrorResult& aRv) {
 159   Sequence<OwningNonNull<MediaStreamTrack>> emptyTrackSeq;
 160   return Constructor(aGlobal, emptyTrackSeq, aRv);
 161 }
 162
 163 /* static */
 164 already_AddRefed<DOMMediaStream> DOMMediaStream::Constructor(
 165     const GlobalObject& aGlobal, const DOMMediaStream& aStream,
 166     ErrorResult& aRv) {
 167   nsTArray<RefPtr<MediaStreamTrack>> tracks;
 168   aStream.GetTracks(tracks);
 169
 170   Sequence<OwningNonNull<MediaStreamTrack>> nonNullTrackSeq;
 171   if (!nonNullTrackSeq.SetLength(tracks.Length(), fallible)) {
 172     MOZ_ASSERT(false);
 173     aRv.Throw(NS_ERROR_OUT_OF_MEMORY);
 174     return nullptr;
 175   }
 176
 177   for (size_t i = 0; i < tracks.Length(); ++i) {
 178     nonNullTrackSeq[i] = tracks[i];
 179   }
 180
 181   return Constructor(aGlobal, nonNullTrackSeq, aRv);
 182 }
 183
 184 /* static */
 185 already_AddRefed<DOMMediaStream> DOMMediaStream::Constructor(
 186     const GlobalObject& aGlobal,
 187     const Sequence<OwningNonNull<MediaStreamTrack>>& aTracks,
 188     ErrorResult& aRv) {
 189   nsCOMPtr<nsPIDOMWindowInner> ownerWindow =
 190       do_QueryInterface(aGlobal.GetAsSupports());
 191   if (!ownerWindow) {
 192     aRv.Throw(NS_ERROR_FAILURE);
 193     return nullptr;
 194   }
 195
 196   auto newStream = MakeRefPtr<DOMMediaStream>(ownerWindow);
 197   for (MediaStreamTrack& track : aTracks) {
 198     newStream->AddTrack(track);
 199   }
 200   return newStream.forget();
 201 }
 202
 203 already_AddRefed<Promise> DOMMediaStream::CountUnderlyingStreams(
 204     const GlobalObject& aGlobal, ErrorResult& aRv) {
 205   nsCOMPtr<nsPIDOMWindowInner> window =
 206       do_QueryInterface(aGlobal.GetAsSupports());
 207   if (!window) {
 208     aRv.Throw(NS_ERROR_UNEXPECTED);
 209     return nullptr;
 210   }
 211
 212   nsCOMPtr<nsIGlobalObject> go = do_QueryInterface(aGlobal.GetAsSupports());
 213   if (!go) {
 214     aRv.Throw(NS_ERROR_UNEXPECTED);
 215     return nullptr;
 216   }
 217
 218   RefPtr<Promise> p = Promise::Create(go, aRv);
 219   if (aRv.Failed()) {
 220     return nullptr;
 221   }
 222
 223   MediaTrackGraph* graph = MediaTrackGraph::GetInstanceIfExists(
 224       window, MediaTrackGraph::REQUEST_DEFAULT_SAMPLE_RATE,
 225       MediaTrackGraph::DEFAULT_OUTPUT_DEVICE);
 226   if (!graph) {
 227     p->MaybeResolve(0);
 228     return p.forget();
 229   }
 230
 231   auto* graphImpl = static_cast<MediaTrackGraphImpl*>(graph);
 232
 233   class Counter : public ControlMessage {
 234    public:
 235     Counter(MediaTrackGraphImpl* aGraph, const RefPtr<Promise>& aPromise)
 236         : ControlMessage(nullptr), mGraph(aGraph), mPromise(aPromise) {
 237       MOZ_ASSERT(NS_IsMainThread());
 238     }
 239
 240     void Run() override {
 241       TRACE("DOMMediaStream::Counter")
 242       uint32_t streams =
 243           mGraph->mTracks.Length() + mGraph->mSuspendedTracks.Length();
 244       mGraph->DispatchToMainThreadStableState(NS_NewRunnableFunction(
 245           "DOMMediaStream::CountUnderlyingStreams (stable state)",
 246           [promise = std::move(mPromise), streams]() mutable {
 247             NS_DispatchToMainThread(NS_NewRunnableFunction(
 248                 "DOMMediaStream::CountUnderlyingStreams",
 249                 [promise = std::move(promise), streams]() {
 250                   promise->MaybeResolve(streams);
 251                 }));
 252           }));
 253     }
 254
 255     // mPromise can only be AddRefed/Released on main thread.
 256     // In case of shutdown, Run() does not run, so we dispatch mPromise to be
 257     // released on main thread here.
 258     void RunDuringShutdown() override {
 259       NS_ReleaseOnMainThread(
 260           "DOMMediaStream::CountUnderlyingStreams::Counter::RunDuringShutdown",
 261           mPromise.forget());
 262     }
 263
 264    private:
 265     // mGraph owns this Counter instance and decides its lifetime.
 266     MediaTrackGraphImpl* mGraph;
 267     RefPtr<Promise> mPromise;
 268   };
 269   graphImpl->AppendMessage(MakeUnique<Counter>(graphImpl, p));
 270
 271   return p.forget();
 272 }
 273
 274 void DOMMediaStream::GetId(nsAString& aID) const { aID = mID; }
 275
 276 void DOMMediaStream::GetAudioTracks(
 277     nsTArray<RefPtr<AudioStreamTrack>>& aTracks) const {
 278   for (const auto& track : mTracks) {
 279     if (AudioStreamTrack* t = track->AsAudioStreamTrack()) {
 280       aTracks.AppendElement(t);
 281     }
 282   }
 283 }
 284
 285 void DOMMediaStream::GetAudioTracks(
 286     nsTArray<RefPtr<MediaStreamTrack>>& aTracks) const {
 287   for (const auto& track : mTracks) {
 288     if (track->AsAudioStreamTrack()) {
 289       aTracks.AppendElement(track);
 290     }
 291   }
 292 }
 293
 294 void DOMMediaStream::GetVideoTracks(
 295     nsTArray<RefPtr<VideoStreamTrack>>& aTracks) const {
 296   for (const auto& track : mTracks) {
 297     if (VideoStreamTrack* t = track->AsVideoStreamTrack()) {
 298       aTracks.AppendElement(t);
 299     }
 300   }
 301 }
 302
 303 void DOMMediaStream::GetVideoTracks(
 304     nsTArray<RefPtr<MediaStreamTrack>>& aTracks) const {
 305   for (const auto& track : mTracks) {
 306     if (track->AsVideoStreamTrack()) {
 307       aTracks.AppendElement(track);
 308     }
 309   }
 310 }
 311
 312 void DOMMediaStream::GetTracks(
 313     nsTArray<RefPtr<MediaStreamTrack>>& aTracks) const {
 314   for (const auto& track : mTracks) {
 315     aTracks.AppendElement(track);
 316   }
 317 }
 318
 319 void DOMMediaStream::AddTrack(MediaStreamTrack& aTrack) {
 320   LOG(LogLevel::Info, ("DOMMediaStream %p Adding track %p (from track %p)",
 321                        this, &aTrack, aTrack.GetTrack()));
 322
 323   if (HasTrack(aTrack)) {
 324     LOG(LogLevel::Debug,
 325         ("DOMMediaStream %p already contains track %p", this, &aTrack));
 326     return;
 327   }
 328
 329   mTracks.AppendElement(&aTrack);
 330
 331   if (!aTrack.Ended()) {
 332     NotifyTrackAdded(&aTrack);
 333   }
 334 }
 335
 336 void DOMMediaStream::RemoveTrack(MediaStreamTrack& aTrack) {
 337   if (static_cast<LogModule*>(gMediaStreamLog)->ShouldLog(LogLevel::Info)) {
 338     if (aTrack.Ended()) {
 339       LOG(LogLevel::Info,
 340           ("DOMMediaStream %p Removing (ended) track %p", this, &aTrack));
 341     } else {
 342       LOG(LogLevel::Info,
 343           ("DOMMediaStream %p Removing track %p (from track %p)", this, &aTrack,
 344            aTrack.GetTrack()));
 345     }
 346   }
 347
 348   if (!mTracks.RemoveElement(&aTrack)) {
 349     LOG(LogLevel::Debug,
 350         ("DOMMediaStream %p does not contain track %p", this, &aTrack));
 351     return;
 352   }
 353
 354   if (!aTrack.Ended()) {
 355     NotifyTrackRemoved(&aTrack);
 356   }
 357 }
 358
 359 already_AddRefed<DOMMediaStream> DOMMediaStream::Clone() {
 360   auto newStream = MakeRefPtr<DOMMediaStream>(GetOwner());
 361
 362   LOG(LogLevel::Info,
 363       ("DOMMediaStream %p created clone %p", this, newStream.get()));
 364
 365   for (const auto& track : mTracks) {
 366     LOG(LogLevel::Debug,
 367         ("DOMMediaStream %p forwarding external track %p to clone %p", this,
 368          track.get(), newStream.get()));
 369     RefPtr<MediaStreamTrack> clone = track->Clone();
 370     newStream->AddTrack(*clone);
 371   }
 372
 373   return newStream.forget();
 374 }
 375
 376 bool DOMMediaStream::Active() const { return mActive; }
 377 bool DOMMediaStream::Audible() const { return mAudible; }
 378
 379 MediaStreamTrack* DOMMediaStream::GetTrackById(const nsAString& aId) const {
 380   for (const auto& track : mTracks) {
 381     nsString id;
 382     track->GetId(id);
 383     if (id == aId) {
 384       return track;
 385     }
 386   }
 387   return nullptr;
 388 }
 389
 390 bool DOMMediaStream::HasTrack(const MediaStreamTrack& aTrack) const {
 391   return mTracks.Contains(&aTrack);
 392 }
 393
 394 void DOMMediaStream::AddTrackInternal(MediaStreamTrack* aTrack) {
 395   LOG(LogLevel::Debug,
 396       ("DOMMediaStream %p Adding owned track %p", this, aTrack));
 397   AddTrack(*aTrack);
 398   DispatchTrackEvent(u"addtrack"_ns, aTrack);
 399 }
 400
 401 void DOMMediaStream::RemoveTrackInternal(MediaStreamTrack* aTrack) {
 402   LOG(LogLevel::Debug,
 403       ("DOMMediaStream %p Removing owned track %p", this, aTrack));
 404   if (!HasTrack(*aTrack)) {
 405     return;
 406   }
 407   RemoveTrack(*aTrack);
 408   DispatchTrackEvent(u"removetrack"_ns, aTrack);
 409 }
 410
 411 already_AddRefed<nsIPrincipal> DOMMediaStream::GetPrincipal() {
 412   if (!GetOwner()) {
 413     return nullptr;
 414   }
 415   nsCOMPtr<nsIPrincipal> principal =
 416       nsGlobalWindowInner::Cast(GetOwner())->GetPrincipal();
 417   for (const auto& t : mTracks) {
 418     if (t->Ended()) {
 419       continue;
 420     }
 421     nsContentUtils::CombineResourcePrincipals(&principal, t->GetPrincipal());
 422   }
 423   return principal.forget();
 424 }
 425
 426 void DOMMediaStream::NotifyActive() {
 427   LOG(LogLevel::Info, ("DOMMediaStream %p NotifyActive(). ", this));
 428
 429   MOZ_ASSERT(mActive);
 430   for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 431     mTrackListeners[i]->NotifyActive();
 432   }
 433 }
 434
 435 void DOMMediaStream::NotifyInactive() {
 436   LOG(LogLevel::Info, ("DOMMediaStream %p NotifyInactive(). ", this));
 437
 438   MOZ_ASSERT(!mActive);
 439   for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 440     mTrackListeners[i]->NotifyInactive();
 441   }
 442 }
 443
 444 void DOMMediaStream::NotifyAudible() {
 445   LOG(LogLevel::Info, ("DOMMediaStream %p NotifyAudible(). ", this));
 446
 447   MOZ_ASSERT(mAudible);
 448   for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 449     mTrackListeners[i]->NotifyAudible();
 450   }
 451 }
 452
 453 void DOMMediaStream::NotifyInaudible() {
 454   LOG(LogLevel::Info, ("DOMMediaStream %p NotifyInaudible(). ", this));
 455
 456   MOZ_ASSERT(!mAudible);
 457   for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 458     mTrackListeners[i]->NotifyInaudible();
 459   }
 460 }
 461
 462 void DOMMediaStream::RegisterTrackListener(TrackListener* aListener) {
 463   MOZ_ASSERT(NS_IsMainThread());
 464
 465   mTrackListeners.AppendElement(aListener);
 466 }
 467
 468 void DOMMediaStream::UnregisterTrackListener(TrackListener* aListener) {
 469   MOZ_ASSERT(NS_IsMainThread());
 470   mTrackListeners.RemoveElement(aListener);
 471 }
 472
 473 void DOMMediaStream::NotifyTrackAdded(const RefPtr<MediaStreamTrack>& aTrack) {
 474   MOZ_ASSERT(NS_IsMainThread());
 475
 476   aTrack->AddConsumer(mPlaybackTrackListener);
 477
 478   for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 479     mTrackListeners[i]->NotifyTrackAdded(aTrack);
 480   }
 481
 482   if (!mActive) {
 483     // Check if we became active.
 484     if (ContainsLiveTracks(mTracks)) {
 485       mActive = true;
 486       NotifyActive();
 487     }
 488   }
 489
 490   if (!mAudible) {
 491     // Check if we became audible.
 492     if (ContainsLiveAudioTracks(mTracks)) {
 493       mAudible = true;
 494       NotifyAudible();
 495     }
 496   }
 497 }
 498
 499 void DOMMediaStream::NotifyTrackRemoved(
 500     const RefPtr<MediaStreamTrack>& aTrack) {
 501   MOZ_ASSERT(NS_IsMainThread());
 502
 503   if (aTrack) {
 504     // aTrack may be null to allow HTMLMediaElement::MozCaptureStream streams
 505     // to be played until the source media element has ended. The source media
 506     // element will then call NotifyTrackRemoved(nullptr) to signal that we can
 507     // go inactive, regardless of the timing of the last track ending.
 508
 509     aTrack->RemoveConsumer(mPlaybackTrackListener);
 510
 511     for (int32_t i = mTrackListeners.Length() - 1; i >= 0; --i) {
 512       mTrackListeners[i]->NotifyTrackRemoved(aTrack);
 513     }
 514
 515     if (!mActive) {
 516       NS_ASSERTION(false, "Shouldn't remove a live track if already inactive");
 517       return;
 518     }
 519   }
 520
 521   if (mAudible) {
 522     // Check if we became inaudible.
 523     if (!ContainsLiveAudioTracks(mTracks)) {
 524       mAudible = false;
 525       NotifyInaudible();
 526     }
 527   }
 528
 529   // Check if we became inactive.
 530   if (!ContainsLiveTracks(mTracks)) {
 531     mActive = false;
 532     NotifyInactive();
 533   }
 534 }
 535
 536 nsresult DOMMediaStream::DispatchTrackEvent(
 537     const nsAString& aName, const RefPtr<MediaStreamTrack>& aTrack) {
 538   MediaStreamTrackEventInit init;
 539   init.mTrack = aTrack;
 540
 541   RefPtr<MediaStreamTrackEvent> event =
 542       MediaStreamTrackEvent::Constructor(this, aName, init);
 543
 544   return DispatchTrustedEvent(event);
 545 }