third_party/libwebrtc/modules/remote_bitrate_estimator/remote_estimator_proxy.cc

   1 /*
   2  *  Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
   3  *
   4  *  Use of this source code is governed by a BSD-style license
   5  *  that can be found in the LICENSE file in the root of the source
   6  *  tree. An additional intellectual property rights grant can be found
   7  *  in the file PATENTS.  All contributing project authors may
   8  *  be found in the AUTHORS file in the root of the source tree.
   9  */
  10
  11 #include "modules/remote_bitrate_estimator/remote_estimator_proxy.h"
  12
  13 #include <algorithm>
  14 #include <cstdint>
  15 #include <limits>
  16 #include <memory>
  17 #include <utility>
  18
  19 #include "absl/types/optional.h"
  20 #include "api/units/data_size.h"
  21 #include "api/units/time_delta.h"
  22 #include "modules/rtp_rtcp/source/rtcp_packet/remote_estimate.h"
  23 #include "modules/rtp_rtcp/source/rtcp_packet/transport_feedback.h"
  24 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
  25 #include "rtc_base/checks.h"
  26 #include "rtc_base/logging.h"
  27 #include "rtc_base/numerics/safe_minmax.h"
  28 #include "system_wrappers/include/clock.h"
  29
  30 namespace webrtc {
  31 namespace {
  32
  33 constexpr TimeDelta kBackWindow = TimeDelta::Millis(500);
  34 constexpr TimeDelta kMinInterval = TimeDelta::Millis(50);
  35 constexpr TimeDelta kMaxInterval = TimeDelta::Millis(250);
  36 constexpr TimeDelta kDefaultInterval = TimeDelta::Millis(100);
  37
  38 TimeDelta GetAbsoluteSendTimeDelta(uint32_t new_sendtime,
  39                                    uint32_t previous_sendtime) {
  40   static constexpr uint32_t kWrapAroundPeriod = 0x0100'0000;
  41   RTC_DCHECK_LT(new_sendtime, kWrapAroundPeriod);
  42   RTC_DCHECK_LT(previous_sendtime, kWrapAroundPeriod);
  43   uint32_t delta = (new_sendtime - previous_sendtime) % kWrapAroundPeriod;
  44   if (delta >= kWrapAroundPeriod / 2) {
  45     // absolute send time wraps around, thus treat deltas larger than half of
  46     // the wrap around period as negative.
  47     delta = (previous_sendtime - new_sendtime) % kWrapAroundPeriod;
  48     return TimeDelta::Micros(int64_t{delta} * -1'000'000 / (1 << 18));
  49   }
  50   return TimeDelta::Micros(int64_t{delta} * 1'000'000 / (1 << 18));
  51 }
  52 }  // namespace
  53
  54 RemoteEstimatorProxy::RemoteEstimatorProxy(
  55     TransportFeedbackSender feedback_sender,
  56     NetworkStateEstimator* network_state_estimator)
  57     : feedback_sender_(std::move(feedback_sender)),
  58       last_process_time_(Timestamp::MinusInfinity()),
  59       network_state_estimator_(network_state_estimator),
  60       media_ssrc_(0),
  61       feedback_packet_count_(0),
  62       packet_overhead_(DataSize::Zero()),
  63       send_interval_(kDefaultInterval),
  64       send_periodic_feedback_(true),
  65       previous_abs_send_time_(0),
  66       abs_send_timestamp_(Timestamp::Zero()),
  67       last_arrival_time_with_abs_send_time_(Timestamp::MinusInfinity()) {
  68   RTC_LOG(LS_INFO)
  69       << "Maximum interval between transport feedback RTCP messages: "
  70       << kMaxInterval;
  71 }
  72
  73 RemoteEstimatorProxy::~RemoteEstimatorProxy() {}
  74
  75 void RemoteEstimatorProxy::MaybeCullOldPackets(int64_t sequence_number,
  76                                                Timestamp arrival_time) {
  77   if (periodic_window_start_seq_ >=
  78           packet_arrival_times_.end_sequence_number() &&
  79       arrival_time - Timestamp::Zero() >= kBackWindow) {
  80     // Start new feedback packet, cull old packets.
  81     packet_arrival_times_.RemoveOldPackets(sequence_number,
  82                                            arrival_time - kBackWindow);
  83   }
  84 }
  85
  86 void RemoteEstimatorProxy::IncomingPacket(const RtpPacketReceived& packet) {
  87   if (packet.arrival_time().IsInfinite()) {
  88     RTC_LOG(LS_WARNING) << "Arrival time not set.";
  89     return;
  90   }
  91
  92   uint16_t seqnum = 0;
  93   absl::optional<FeedbackRequest> feedback_request;
  94   if (!packet.GetExtension<TransportSequenceNumber>(&seqnum) &&
  95       !packet.GetExtension<TransportSequenceNumberV2>(&seqnum,
  96                                                       &feedback_request)) {
  97     // This function expected to be called only for packets that have
  98     // TransportSequenceNumber rtp header extension, however malformed RTP
  99     // packet may contain unparsable TransportSequenceNumber.
 100     RTC_DCHECK(packet.HasExtension<TransportSequenceNumber>() ||
 101                packet.HasExtension<TransportSequenceNumberV2>())
 102         << " Expected transport sequence number.";
 103     return;
 104   }
 105
 106   MutexLock lock(&lock_);
 107   send_periodic_feedback_ = packet.HasExtension<TransportSequenceNumber>();
 108
 109   media_ssrc_ = packet.Ssrc();
 110   int64_t seq = unwrapper_.Unwrap(seqnum);
 111
 112   if (send_periodic_feedback_) {
 113     MaybeCullOldPackets(seq, packet.arrival_time());
 114
 115     if (!periodic_window_start_seq_ || seq < *periodic_window_start_seq_) {
 116       periodic_window_start_seq_ = seq;
 117     }
 118   }
 119
 120   // We are only interested in the first time a packet is received.
 121   if (packet_arrival_times_.has_received(seq)) {
 122     return;
 123   }
 124
 125   packet_arrival_times_.AddPacket(seq, packet.arrival_time());
 126
 127   // Limit the range of sequence numbers to send feedback for.
 128   if (periodic_window_start_seq_ <
 129       packet_arrival_times_.begin_sequence_number()) {
 130     periodic_window_start_seq_ = packet_arrival_times_.begin_sequence_number();
 131   }
 132
 133   if (feedback_request.has_value()) {
 134     // Send feedback packet immediately.
 135     SendFeedbackOnRequest(seq, *feedback_request);
 136   }
 137
 138   absl::optional<uint32_t> absolute_send_time_24bits =
 139       packet.GetExtension<AbsoluteSendTime>();
 140   if (network_state_estimator_ && absolute_send_time_24bits.has_value()) {
 141     PacketResult packet_result;
 142     packet_result.receive_time = packet.arrival_time();
 143     if (packet.arrival_time() - last_arrival_time_with_abs_send_time_ <
 144         TimeDelta::Seconds(10)) {
 145       abs_send_timestamp_ += GetAbsoluteSendTimeDelta(
 146           *absolute_send_time_24bits, previous_abs_send_time_);
 147     } else {
 148       abs_send_timestamp_ = packet.arrival_time();
 149     }
 150     last_arrival_time_with_abs_send_time_ = packet.arrival_time();
 151     previous_abs_send_time_ = *absolute_send_time_24bits;
 152     packet_result.sent_packet.send_time = abs_send_timestamp_;
 153     packet_result.sent_packet.size =
 154         DataSize::Bytes(packet.size()) + packet_overhead_;
 155     packet_result.sent_packet.sequence_number = seq;
 156     network_state_estimator_->OnReceivedPacket(packet_result);
 157   }
 158 }
 159
 160 TimeDelta RemoteEstimatorProxy::Process(Timestamp now) {
 161   MutexLock lock(&lock_);
 162   if (!send_periodic_feedback_) {
 163     // If TransportSequenceNumberV2 has been received in one packet,
 164     // PeriodicFeedback is disabled for the rest of the call.
 165     return TimeDelta::PlusInfinity();
 166   }
 167   Timestamp next_process_time = last_process_time_ + send_interval_;
 168   if (now >= next_process_time) {
 169     last_process_time_ = now;
 170     SendPeriodicFeedbacks();
 171     return send_interval_;
 172   }
 173
 174   return next_process_time - now;
 175 }
 176
 177 void RemoteEstimatorProxy::OnBitrateChanged(int bitrate_bps) {
 178   // TwccReportSize = Ipv4(20B) + UDP(8B) + SRTP(10B) +
 179   // AverageTwccReport(30B)
 180   // TwccReport size at 50ms interval is 24 byte.
 181   // TwccReport size at 250ms interval is 36 byte.
 182   // AverageTwccReport = (TwccReport(50ms) + TwccReport(250ms)) / 2
 183   constexpr DataSize kTwccReportSize = DataSize::Bytes(20 + 8 + 10 + 30);
 184   constexpr DataRate kMinTwccRate = kTwccReportSize / kMaxInterval;
 185
 186   // Let TWCC reports occupy 5% of total bandwidth.
 187   DataRate twcc_bitrate = DataRate::BitsPerSec(0.05 * bitrate_bps);
 188
 189   // Check upper send_interval bound by checking bitrate to avoid overflow when
 190   // dividing by small bitrate, in particular avoid dividing by zero bitrate.
 191   TimeDelta send_interval =
 192       twcc_bitrate <= kMinTwccRate
 193           ? kMaxInterval
 194           : std::max(kTwccReportSize / twcc_bitrate, kMinInterval);
 195
 196   MutexLock lock(&lock_);
 197   send_interval_ = send_interval;
 198 }
 199
 200 void RemoteEstimatorProxy::SetTransportOverhead(DataSize overhead_per_packet) {
 201   MutexLock lock(&lock_);
 202   packet_overhead_ = overhead_per_packet;
 203 }
 204
 205 void RemoteEstimatorProxy::SendPeriodicFeedbacks() {
 206   // `periodic_window_start_seq_` is the first sequence number to include in
 207   // the current feedback packet. Some older may still be in the map, in case
 208   // a reordering happens and we need to retransmit them.
 209   if (!periodic_window_start_seq_)
 210     return;
 211
 212   std::unique_ptr<rtcp::RemoteEstimate> remote_estimate;
 213   if (network_state_estimator_) {
 214     absl::optional<NetworkStateEstimate> state_estimate =
 215         network_state_estimator_->GetCurrentEstimate();
 216     if (state_estimate) {
 217       remote_estimate = std::make_unique<rtcp::RemoteEstimate>();
 218       remote_estimate->SetEstimate(state_estimate.value());
 219     }
 220   }
 221
 222   int64_t packet_arrival_times_end_seq =
 223       packet_arrival_times_.end_sequence_number();
 224   while (periodic_window_start_seq_ < packet_arrival_times_end_seq) {
 225     auto feedback_packet = MaybeBuildFeedbackPacket(
 226         /*include_timestamps=*/true, *periodic_window_start_seq_,
 227         packet_arrival_times_end_seq,
 228         /*is_periodic_update=*/true);
 229
 230     if (feedback_packet == nullptr) {
 231       break;
 232     }
 233
 234     RTC_DCHECK(feedback_sender_ != nullptr);
 235
 236     std::vector<std::unique_ptr<rtcp::RtcpPacket>> packets;
 237     if (remote_estimate) {
 238       packets.push_back(std::move(remote_estimate));
 239     }
 240     packets.push_back(std::move(feedback_packet));
 241
 242     feedback_sender_(std::move(packets));
 243     // Note: Don't erase items from packet_arrival_times_ after sending, in
 244     // case they need to be re-sent after a reordering. Removal will be
 245     // handled by OnPacketArrival once packets are too old.
 246   }
 247 }
 248
 249 void RemoteEstimatorProxy::SendFeedbackOnRequest(
 250     int64_t sequence_number,
 251     const FeedbackRequest& feedback_request) {
 252   if (feedback_request.sequence_count == 0) {
 253     return;
 254   }
 255
 256   int64_t first_sequence_number =
 257       sequence_number - feedback_request.sequence_count + 1;
 258
 259   auto feedback_packet = MaybeBuildFeedbackPacket(
 260       feedback_request.include_timestamps, first_sequence_number,
 261       sequence_number + 1, /*is_periodic_update=*/false);
 262
 263   // Even though this is called when a packet has just been added,
 264   // no feedback may be produced when that new packet is too old.
 265   if (feedback_packet == nullptr) {
 266     return;
 267   }
 268
 269   // Clear up to the first packet that is included in this feedback packet.
 270   packet_arrival_times_.EraseTo(first_sequence_number);
 271
 272   RTC_DCHECK(feedback_sender_ != nullptr);
 273   std::vector<std::unique_ptr<rtcp::RtcpPacket>> packets;
 274   packets.push_back(std::move(feedback_packet));
 275   feedback_sender_(std::move(packets));
 276 }
 277
 278 std::unique_ptr<rtcp::TransportFeedback>
 279 RemoteEstimatorProxy::MaybeBuildFeedbackPacket(
 280     bool include_timestamps,
 281     int64_t begin_sequence_number_inclusive,
 282     int64_t end_sequence_number_exclusive,
 283     bool is_periodic_update) {
 284   RTC_DCHECK_LT(begin_sequence_number_inclusive, end_sequence_number_exclusive);
 285
 286   int64_t start_seq =
 287       packet_arrival_times_.clamp(begin_sequence_number_inclusive);
 288
 289   int64_t end_seq = packet_arrival_times_.clamp(end_sequence_number_exclusive);
 290
 291   // Create the packet on demand, as it's not certain that there are packets
 292   // in the range that have been received.
 293   std::unique_ptr<rtcp::TransportFeedback> feedback_packet;
 294
 295   int64_t next_sequence_number = begin_sequence_number_inclusive;
 296
 297   for (int64_t seq = start_seq; seq < end_seq; ++seq) {
 298     PacketArrivalTimeMap::PacketArrivalTime packet =
 299         packet_arrival_times_.FindNextAtOrAfter(seq);
 300     seq = packet.sequence_number;
 301     if (seq >= end_seq) {
 302       break;
 303     }
 304
 305     if (feedback_packet == nullptr) {
 306       feedback_packet =
 307           std::make_unique<rtcp::TransportFeedback>(include_timestamps);
 308       feedback_packet->SetMediaSsrc(media_ssrc_);
 309
 310       // It should be possible to add `seq` to this new `feedback_packet`,
 311       // If difference between `seq` and `begin_sequence_number_inclusive`,
 312       // is too large, discard reporting too old missing packets.
 313       static constexpr int kMaxMissingSequenceNumbers = 0x7FFE;
 314       int64_t base_sequence_number = std::max(begin_sequence_number_inclusive,
 315                                               seq - kMaxMissingSequenceNumbers);
 316
 317       // Base sequence number is the expected first sequence number. This is
 318       // known, but we might not have actually received it, so the base time
 319       // shall be the time of the first received packet in the feedback.
 320       feedback_packet->SetBase(static_cast<uint16_t>(base_sequence_number),
 321                                packet.arrival_time);
 322       feedback_packet->SetFeedbackSequenceNumber(feedback_packet_count_++);
 323
 324       if (!feedback_packet->AddReceivedPacket(static_cast<uint16_t>(seq),
 325                                               packet.arrival_time)) {
 326         // Could not add a single received packet to the feedback.
 327         RTC_DCHECK_NOTREACHED()
 328             << "Failed to create an RTCP transport feedback with base sequence "
 329                "number "
 330             << base_sequence_number << " and 1st received " << seq;
 331         periodic_window_start_seq_ = seq;
 332         return nullptr;
 333       }
 334     } else {
 335       if (!feedback_packet->AddReceivedPacket(static_cast<uint16_t>(seq),
 336                                               packet.arrival_time)) {
 337         // Could not add timestamp, feedback packet might be full. Return and
 338         // try again with a fresh packet.
 339         break;
 340       }
 341     }
 342
 343     next_sequence_number = seq + 1;
 344   }
 345   if (is_periodic_update) {
 346     periodic_window_start_seq_ = next_sequence_number;
 347   }
 348   return feedback_packet;
 349 }
 350
 351 }  // namespace webrtc