widget/TouchResampler.h

   1 /* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
   2 /* vim: set ts=8 sts=2 et sw=2 tw=80: */
   3 /* This Source Code Form is subject to the terms of the Mozilla Public
   4  * License, v. 2.0. If a copy of the MPL was not distributed with this
   5  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
   6
   7 #ifndef mozilla_widget_TouchResampler_h
   8 #define mozilla_widget_TouchResampler_h
   9
  10 #include <queue>
  11 #include <unordered_map>
  12
  13 #include "mozilla/Maybe.h"
  14 #include "mozilla/TimeStamp.h"
  15 #include "InputData.h"
  16
  17 namespace mozilla {
  18 namespace widget {
  19
  20 /**
  21  * De-jitters touch motions by resampling (interpolating or extrapolating) touch
  22  * positions for the vsync timestamp.
  23  *
  24  * Touch resampling improves the touch panning experience on devices where touch
  25  * positions are sampled at a rate that's not an integer multiple of the display
  26  * refresh rate, for example 100Hz touch sampling on a 60Hz display: Without
  27  * resampling, we would alternate between taking one touch sample or two touch
  28  * samples into account each frame, creating a jittery motion ("small step, big
  29  * step, small step, big step").
  30  * Intended for use on Android, where both touch events and vsync notifications
  31  * arrive on the same thread, the Java UI thread.
  32  * This class is not thread safe.
  33  *
  34  * TouchResampler operates in the following way:
  35  *
  36  * Original events are fed into ProcessEvent().
  37  * Outgoing events (potentially resampled for resampling) are added to a queue
  38  * and can be consumed by calling ConsumeOutgoingEvents(). Touch events which
  39  * are not touch move events are forwarded instantly and not resampled. Only
  40  * touch move events are resampled. Whenever a touch move event is received, it
  41  * gets delayed until NotifyFrame() is called, at which point it is resampled
  42  * into a resampled version for the given frame timestamp, and added to the
  43  * outgoing queue. If no touch move event is received between two consecutive
  44  * frames, this is treated as a stop in the touch motion. If the last outgoing
  45  * event was an resampled touch move event, we return back to the non-resampled
  46  * state by emitting a copy of the last original touch move event, which has
  47  * unmodified position data. Touch events which are not touch move events also
  48  * force a return to the non-resampled state before they are moved to the
  49  * outgoing queue.
  50  */
  51 class TouchResampler final {
  52  public:
  53   // Feed a touch event into the interpolater. Returns an ID that can be used to
  54   // match outgoing events to this incoming event, to track data associated with
  55   // this event.
  56   uint64_t ProcessEvent(MultiTouchInput&& aInput);
  57
  58   // Emit events, potentially resampled, for this timestamp. The events are put
  59   // into the outgoing queue. May not emit any events if there's no update.
  60   void NotifyFrame(const TimeStamp& aTimeStamp);
  61
  62   // Returns true between the start and the end of a touch gesture. During this
  63   // time, the caller should keep itself registered with the system frame
  64   // callback mechanism, so that NotifyFrame() can be called on every frame.
  65   // (Otherwise, if we only registered the callback after receiving a touch move
  66   // event, the frame callback might be delayed by a full frame.)
  67   bool InTouchingState() const { return mCurrentTouches.HasTouch(); }
  68
  69   struct OutgoingEvent {
  70     // The event, potentially modified from the original for resampling.
  71     MultiTouchInput mEvent;
  72
  73     // Some(eventId) if this event is a modified version of an original event,
  74     // Nothing() if this is an extra event.
  75     Maybe<uint64_t> mEventId;
  76   };
  77
  78   // Returns the outgoing events that were produced since the last call.
  79   // No event IDs will be skipped. Returns at least one outgoing event for each
  80   // incoming event (possibly after a delay), and potential extra events with
  81   // no originating event ID.
  82   // Outgoing events should be consumed after every call to ProcessEvent() and
  83   // after every call to NotifyFrame().
  84   std::queue<OutgoingEvent> ConsumeOutgoingEvents() {
  85     return std::move(mOutgoingEvents);
  86   }
  87
  88  private:
  89   // Add the event to the outgoing queue.
  90   void EmitEvent(MultiTouchInput&& aInput, uint64_t aEventId) {
  91     mLastEmittedEventTime = aInput.mTimeStamp;
  92     mOutgoingEvents.push(OutgoingEvent{std::move(aInput), Some(aEventId)});
  93   }
  94
  95   // Emit an event that does not correspond to an incoming event.
  96   void EmitExtraEvent(MultiTouchInput&& aInput) {
  97     mLastEmittedEventTime = aInput.mTimeStamp;
  98     mOutgoingEvents.push(OutgoingEvent{std::move(aInput), Nothing()});
  99   }
 100
 101   // Move any touch move events that we deferred for resampling to the outgoing
 102   // queue unmodified, leaving mDeferredTouchMoveEvents empty.
 103   void FlushDeferredTouchMoveEventsUnresampled();
 104
 105   // Must only be called if mInResampledState is true and
 106   // mDeferredTouchMoveEvents is empty. Emits mOriginalOfResampledTouchMove,
 107   // with a potentially adjusted timestamp for correct ordering.
 108   void ReturnToNonResampledState();
 109
 110   // Takes historical touch data from mRemainingTouchData and prepends it to the
 111   // data in aInput.
 112   void PrependLeftoverHistoricalData(MultiTouchInput* aInput);
 113
 114   struct DataPoint {
 115     TimeStamp mTimeStamp;
 116     ScreenIntPoint mPosition;
 117   };
 118
 119   struct TouchInfo {
 120     void Update(const SingleTouchData& aTouch, const TimeStamp& aEventTime);
 121     ScreenIntPoint ResampleAtTime(const ScreenIntPoint& aLastObservedPosition,
 122                                   const TimeStamp& aTimeStamp);
 123
 124     int32_t mIdentifier = 0;
 125     Maybe<DataPoint> mBaseDataPoint;
 126     Maybe<DataPoint> mLatestDataPoint;
 127   };
 128
 129   struct CurrentTouches {
 130     void UpdateFromEvent(const MultiTouchInput& aInput);
 131     bool HasTouch() const { return !mTouches.IsEmpty(); }
 132     TimeStamp LatestDataPointTime() { return mLatestDataPointTime; }
 133
 134     ScreenIntPoint ResampleTouchPositionAtTime(
 135         int32_t aIdentifier, const ScreenIntPoint& aLastObservedPosition,
 136         const TimeStamp& aTimeStamp);
 137
 138     void ClearDataPoints() {
 139       for (auto& touch : mTouches) {
 140         touch.mBaseDataPoint = Nothing();
 141         touch.mLatestDataPoint = Nothing();
 142       }
 143     }
 144
 145    private:
 146     nsTArray<TouchInfo>::iterator TouchByIdentifier(int32_t aIdentifier);
 147
 148     nsTArray<TouchInfo> mTouches;
 149     TimeStamp mLatestDataPointTime;
 150   };
 151
 152   // The current touch positions with historical data points. This data only
 153   // contains original non-resampled positions from the incoming touch events.
 154   CurrentTouches mCurrentTouches;
 155
 156   // Incoming touch move events are stored here until NotifyFrame is called.
 157   std::queue<std::pair<MultiTouchInput, uint64_t>> mDeferredTouchMoveEvents;
 158
 159   // Stores any touch samples that were not included in the last emitted touch
 160   // move event because they were in the future compared to the emitted event's
 161   // timestamp. These data points should be prepended to the historical data of
 162   // the next emitted touch move evnt.
 163   // Can only be non-empty if mInResampledState is true.
 164   std::unordered_map<int32_t, nsTArray<SingleTouchData::HistoricalTouchData>>
 165       mRemainingTouchData;
 166
 167   // If we're in an resampled state, because the last outgoing event was a
 168   // resampled touch move event, then this contains a copy of the unresampled,
 169   // original touch move event.
 170   // Some() iff mInResampledState is true.
 171   Maybe<MultiTouchInput> mOriginalOfResampledTouchMove;
 172
 173   // The stream of outgoing events that can be consumed by our caller.
 174   std::queue<OutgoingEvent> mOutgoingEvents;
 175
 176   // The timestamp of the event that was emitted most recently, or the null
 177   // timestamp if no event has been emitted yet.
 178   TimeStamp mLastEmittedEventTime;
 179
 180   uint64_t mNextEventId = 0;
 181
 182   // True if the last outgoing event was a touch move event with an resampled
 183   // position. We only want to stay in this state as long as a continuous stream
 184   // of touch move events is coming in.
 185   bool mInResampledState = false;
 186 };
 187
 188 }  // namespace widget
 189 }  // namespace mozilla
 190
 191 #endif  // mozilla_widget_TouchResampler_h