Roll src/third_party/WebKit 432d75f:a15c644 (svn 200751:200754)
[chromium-blink-merge.git] / ui / chromeos / touch_exploration_controller.h
blobea46fd5647e9c5aef4067e6277aadf228070d421
1 // Copyright 2014 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #ifndef UI_CHROMEOS_TOUCH_EXPLORATION_CONTROLLER_H_
6 #define UI_CHROMEOS_TOUCH_EXPLORATION_CONTROLLER_H_
8 #include "base/time/tick_clock.h"
9 #include "base/timer/timer.h"
10 #include "base/values.h"
11 #include "ui/chromeos/ui_chromeos_export.h"
12 #include "ui/events/event.h"
13 #include "ui/events/event_rewriter.h"
14 #include "ui/events/gesture_detection/gesture_detector.h"
15 #include "ui/events/gestures/gesture_provider_aura.h"
16 #include "ui/gfx/geometry/point.h"
18 namespace aura {
19 class Window;
22 namespace ui {
24 class Event;
25 class EventHandler;
26 class GestureEvent;
27 class GestureProviderAura;
28 class TouchEvent;
30 // A delegate to handle commands in response to detected accessibility gesture
31 // events.
32 class TouchExplorationControllerDelegate {
33 public:
34 virtual ~TouchExplorationControllerDelegate() {}
36 // Takes an int from 0.0 to 100.0 that indicates the percent the volume
37 // should be set to.
38 virtual void SetOutputLevel(int volume) = 0;
40 // Silences spoken feedback.
41 virtual void SilenceSpokenFeedback() = 0;
43 // This function should be called when the volume adjust earcon should be
44 // played
45 virtual void PlayVolumeAdjustEarcon() = 0;
47 // This function should be called when the passthrough earcon should be
48 // played.
49 virtual void PlayPassthroughEarcon() = 0;
51 // This function should be called when the exit screen earcon should be
52 // played.
53 virtual void PlayExitScreenEarcon() = 0;
55 // This function should be called when the enter screen earcon should be
56 // played.
57 virtual void PlayEnterScreenEarcon() = 0;
60 // TouchExplorationController is used in tandem with "Spoken Feedback" to
61 // make the touch UI accessible. Gestures performed in the middle of the screen
62 // are mapped to accessibility key shortcuts while gestures performed on the
63 // edge of the screen can change settings.
65 // ** Short version **
67 // At a high-level, single-finger events are used for accessibility -
68 // exploring the screen gets turned into mouse moves (which can then be
69 // spoken by an accessibility service running), a single tap while the user
70 // is in touch exploration or a double-tap simulates a click, and gestures
71 // can be used to send high-level accessibility commands. For example, a swipe
72 // right would correspond to the keyboard short cut shift+search+right.
73 // Swipes with up to four fingers are also mapped to commands. Slide
74 // gestures performed on the edge of the screen can change settings
75 // continuously. For example, sliding a finger along the right side of the
76 // screen will change the volume. When a user double taps and holds with one
77 // finger, the finger is passed through as if accessibility was turned off. If
78 // the user taps the screen with two fingers, the user can silence spoken
79 // feedback if it is playing.
81 // ** Long version **
83 // Here are the details of the implementation:
85 // When the first touch is pressed, a 300 ms grace period timer starts.
87 // If the user keeps their finger down for more than 300 ms and doesn't
88 // perform a supported accessibility gesture in that time (e.g. swipe right),
89 // they enter touch exploration mode, and all movements are translated into
90 // synthesized mouse move events.
92 // Also, if the user moves their single finger outside a certain slop region
93 // (without performing a gesture), they enter touch exploration mode earlier
94 // than 300 ms.
96 // If the user taps and releases their finger, after 300 ms from the initial
97 // touch, a single mouse move is fired.
99 // While in touch exploration mode, the user can perform a single tap
100 // if the user releases their finger and taps before 300 ms passes.
101 // This will result in a click on the last successful touch exploration
102 // location. This allows the user to perform a single tap
103 // anywhere to activate it.
105 // The user can perform swipe gestures in one of the four cardinal directions
106 // which will be interpreted and used to control the UI. All gestures will only
107 // be registered if the fingers move outside the slop, and all fingers will only
108 // be registered if they are completed within the grace period. If a single
109 // finger gesture fails to be completed within the grace period, the state
110 // changes to touch exploration mode. If a multi finger gesture fails to be
111 // completed within the grace period, the user must lift all fingers before
112 // completing any more actions.
114 // If the user double-taps, the second tap is passed through, allowing the
115 // user to click - however, the double-tap location is changed to the location
116 // of the last successful touch exploration - that allows the user to explore
117 // anywhere on the screen, hear its description, then double-tap anywhere
118 // to activate it.
120 // If the user double taps and holds, any event from that finger is passed
121 // through. These events are passed through with an offset such that the first
122 // touch is offset to be at the location of the last touch exploration
123 // location, and every following event is offset by the same amount.
125 // If any other fingers are added or removed, they are ignored. Once the
126 // passthrough finger is released, passthrough stops and the user is reset
127 // to no fingers down state.
129 // If the user enters touch exploration mode, they can click without lifting
130 // their touch exploration finger by tapping anywhere else on the screen with
131 // a second finger, while the touch exploration finger is still pressed.
133 // Once touch exploration mode has been activated, it remains in that mode until
134 // all fingers have been released.
136 // If the user places a finger on the edge of the screen and moves their finger
137 // past slop, a slide gesture is performed. The user can then slide one finger
138 // along an edge of the screen and continuously control a setting. Once the user
139 // enters this state, the boundaries that define an edge expand so that the user
140 // can now adjust the setting within a slightly bigger width along the screen.
141 // If the user exits this area without lifting their finger, they will not be
142 // able to perform any actions, however if they keep their finger down and
143 // return to the "hot edge," then they can still adjust the setting. In order to
144 // perform other touch accessibility movements, the user must lift their finger.
145 // If additional fingers are added while in this state, the user will transition
146 // to passthrough.
148 // Currently, only the right edge is mapped to control the volume. Volume
149 // control along the edge of the screen is directly proportional to where the
150 // user's finger is located on the screen. The top right corner of the screen
151 // automatically sets the volume to 100% and the bottome right corner of the
152 // screen automatically sets the volume to 0% once the user has moved past slop.
154 // If the user taps the screen with two fingers and lifts both fingers before
155 // the grace period has passed, spoken feedback is silenced.
157 // The user can also enter passthrough by placing a finger on one of the bottom
158 // corners of the screen until an earcon sounds. After the earcon sounds, the
159 // user is in passthrough so all subsequent fingers placed on the screen will be
160 // passed through. Once the finger in the corner has been released, the state
161 // will switch to wait for no fingers.
163 // The caller is expected to retain ownership of instances of this class and
164 // destroy them before |root_window| is destroyed.
165 class UI_CHROMEOS_EXPORT TouchExplorationController
166 : public ui::EventRewriter,
167 public ui::GestureProviderAuraClient {
168 public:
169 explicit TouchExplorationController(
170 aura::Window* root_window,
171 ui::TouchExplorationControllerDelegate* delegate);
172 ~TouchExplorationController() override;
174 private:
175 friend class TouchExplorationControllerTestApi;
177 // Overridden from ui::EventRewriter
178 ui::EventRewriteStatus RewriteEvent(
179 const ui::Event& event,
180 scoped_ptr<ui::Event>* rewritten_event) override;
181 ui::EventRewriteStatus NextDispatchEvent(
182 const ui::Event& last_event,
183 scoped_ptr<ui::Event>* new_event) override;
185 // Event handlers based on the current state - see State, below.
186 ui::EventRewriteStatus InNoFingersDown(
187 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
188 ui::EventRewriteStatus InSingleTapPressed(
189 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
190 ui::EventRewriteStatus InSingleTapOrTouchExploreReleased(
191 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
192 ui::EventRewriteStatus InDoubleTapPending(
193 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
194 ui::EventRewriteStatus InTouchReleasePending(
195 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
196 ui::EventRewriteStatus InTouchExploration(
197 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
198 ui::EventRewriteStatus InCornerPassthrough(
199 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
200 ui::EventRewriteStatus InOneFingerPassthrough(
201 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
202 ui::EventRewriteStatus InGestureInProgress(
203 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
204 ui::EventRewriteStatus InTouchExploreSecondPress(
205 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
206 ui::EventRewriteStatus InWaitForNoFingers(
207 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
208 ui::EventRewriteStatus InSlideGesture(
209 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
210 ui::EventRewriteStatus InTwoFingerTap(
211 const ui::TouchEvent& event, scoped_ptr<ui::Event>* rewritten_event);
213 // Returns the current time of the tick clock.
214 base::TimeDelta Now();
216 // This timer is started every time we get the first press event, and
217 // it fires after the double-click timeout elapses (300 ms by default).
218 // If the user taps and releases within 300 ms and doesn't press again,
219 // we treat that as a single mouse move (touch exploration) event.
220 void StartTapTimer();
221 void OnTapTimerFired();
223 // This timer is started every timer we get the first press event and the
224 // finger is in the corner of the screen.
225 // It fires after the corner passthrough delay elapses. If the
226 // user is still in the corner by the time this timer fires, all subsequent
227 // fingers added on the screen will be passed through.
228 void OnPassthroughTimerFired();
230 // Dispatch a new event outside of the event rewriting flow.
231 void DispatchEvent(ui::Event* event);
233 // Overridden from GestureProviderAuraClient.
235 // The gesture provider keeps track of all the touch events after
236 // the user moves fast enough to trigger a gesture. After the user
237 // completes their gesture, this method will decide what keyboard
238 // input their gesture corresponded to.
239 void OnGestureEvent(ui::GestureEvent* gesture) override;
241 // Process the gesture events that have been created.
242 void ProcessGestureEvents();
244 void OnSwipeEvent(ui::GestureEvent* swipe_gesture);
246 void SideSlideControl(ui::GestureEvent* gesture);
248 // Dispatches the keyboard short cut Shift+Search+<arrow key>
249 // outside the event rewritting flow.
250 void DispatchShiftSearchKeyEvent(const ui::KeyboardCode third_key);
252 // Binds DispatchShiftSearchKeyEvent to a specific third key.
253 base::Closure BindShiftSearchKeyEvent(const ui::KeyboardCode third_key);
255 // Dispatches a single key with the given flags.
256 void DispatchKeyWithFlags(const ui::KeyboardCode key, int flags);
258 // Binds DispatchKeyWithFlags to a specific key and flags.
259 base::Closure BindKeyEventWithFlags(const ui::KeyboardCode key, int flags);
261 scoped_ptr<ui::Event> CreateMouseMoveEvent(const gfx::PointF& location,
262 int flags);
264 void EnterTouchToMouseMode();
266 void PlaySoundForTimer();
268 // Some constants used in touch_exploration_controller:
270 // Within this many dips of the screen edge, the release event generated will
271 // reset the state to NoFingersDown.
272 const float kLeavingScreenEdge = 6;
274 // Swipe/scroll gestures within these bounds (in DIPs) will change preset
275 // settings.
276 const float kMaxDistanceFromEdge = 75;
278 // After a slide gesture has been triggered, if the finger is still within
279 // these bounds (in DIPs), the preset settings will still change.
280 const float kSlopDistanceFromEdge = kMaxDistanceFromEdge + 40;
282 // The split tap slop is a bit more generous since keeping two
283 // fingers in place is a bit harder.
284 float GetSplitTapTouchSlop();
286 enum State {
287 // No fingers are down and no events are pending.
288 NO_FINGERS_DOWN,
290 // A single finger is down, but we're not yet sure if this is going
291 // to be touch exploration or something else.
292 SINGLE_TAP_PRESSED,
294 // The user pressed and released a single finger - a tap - but we have
295 // to wait until the end of the grace period to allow the user to tap the
296 // second time. If the second tap doesn't occurs within the grace period,
297 // we dispatch a mouse move at the location of the first tap.
298 SINGLE_TAP_RELEASED,
300 // The user was in touch explore mode and released the finger.
301 // If another touch press occurs within the grace period, a single
302 // tap click occurs. This state differs from SINGLE_TAP_RELEASED
303 // in that if a second tap doesn't occur within the grace period,
304 // there is no mouse move dispatched.
305 TOUCH_EXPLORE_RELEASED,
307 // The user tapped once, and before the grace period expired, pressed
308 // one finger down to begin a double-tap, but has not released it yet.
309 // This could become passthrough, so no touch press is dispatched yet.
310 DOUBLE_TAP_PENDING,
312 // The user was doing touch exploration, started split tap, but lifted the
313 // touch exploration finger. Once they remove all fingers, a touch release
314 // will go through.
315 TOUCH_RELEASE_PENDING,
317 // We're in touch exploration mode. Anything other than the first finger
318 // is ignored, and movements of the first finger are rewritten as mouse
319 // move events. This mode is entered if a single finger is pressed and
320 // after the grace period the user hasn't added a second finger or
321 // moved the finger outside of the slop region. We'll stay in this
322 // mode until all fingers are lifted.
323 TOUCH_EXPLORATION,
325 // If the user moves their finger faster than the threshold velocity after a
326 // single tap, the touch events that follow will be translated into gesture
327 // events. If the user successfully completes a gesture within the grace
328 // period, the gesture will be interpreted and used to control the UI via
329 // discrete actions - currently by synthesizing key events corresponding to
330 // each gesture Otherwise, the collected gestures are discarded and the
331 // state changes to touch_exploration.
332 GESTURE_IN_PROGRESS,
334 // The user was in touch exploration, but has placed down another finger.
335 // If the user releases the second finger, a touch press and release
336 // will go through at the last touch explore location. If the user
337 // releases the touch explore finger, the touch press and release will
338 // still go through once the split tap finger is also lifted. If any
339 // fingers pressed past the first two, the touch press is cancelled and
340 // the user enters the wait state for the fingers to be removed.
341 TOUCH_EXPLORE_SECOND_PRESS,
343 // After the user double taps and holds with a single finger, all events
344 // for that finger are passed through, displaced by an offset. Adding
345 // extra fingers has no effect. This state is left when the user removes
346 // all fingers.
347 ONE_FINGER_PASSTHROUGH,
349 // If the user has pressed and held down the left corner past long press,
350 // then as long as they are holding the corner, all subsequent fingers
351 // registered will be in passthrough.
352 CORNER_PASSTHROUGH,
354 // If the user added another finger in SINGLE_TAP_PRESSED, or if the user
355 // has multiple fingers fingers down in any other state between
356 // passthrough, touch exploration, and gestures, they must release
357 // all fingers before completing any more actions. This state is
358 // generally useful for developing new features, because it creates a
359 // simple way to handle a dead end in user flow.
360 WAIT_FOR_NO_FINGERS,
362 // If the user is within the given bounds from an edge of the screen, not
363 // including corners, then the resulting movements will be interpreted as
364 // slide gestures.
365 SLIDE_GESTURE,
367 // If the user taps the screen with two fingers and releases both fingers
368 // before the grace period has passed, spoken feedback will be silenced.
369 TWO_FINGER_TAP,
372 enum ScreenLocation {
373 // Hot "edges" of the screen are each represented by a respective bit.
374 NO_EDGE = 0,
375 RIGHT_EDGE = 1 << 0,
376 TOP_EDGE = 1 << 1,
377 LEFT_EDGE = 1 << 2,
378 BOTTOM_EDGE = 1 << 3,
379 BOTTOM_LEFT_CORNER = LEFT_EDGE | BOTTOM_EDGE,
380 BOTTOM_RIGHT_CORNER = RIGHT_EDGE | BOTTOM_EDGE,
383 // Given a point, if it is within the given bounds of an edge, returns the
384 // edge. If it is within the given bounds of two edges, returns an int with
385 // both bits that represent the respective edges turned on. Otherwise returns
386 // SCREEN_CENTER.
387 int FindEdgesWithinBounds(gfx::Point point, float bounds);
389 // Set the state and modifies any variables related to the state change.
390 // (e.g. resetting the gesture provider).
391 void SetState(State new_state, const char* function_name);
393 void VlogState(const char* function_name);
395 void VlogEvent(const ui::TouchEvent& event, const char* function_name);
397 // Gets enum name from integer value.
398 const char* EnumStateToString(State state);
400 // Maps each single/multi finger swipe to the function that dispatches
401 // the corresponding key events.
402 void InitializeSwipeGestureMaps();
404 aura::Window* root_window_;
406 // Handles volume control. Not owned.
407 ui::TouchExplorationControllerDelegate* delegate_;
409 // A set of touch ids for fingers currently touching the screen.
410 std::vector<int> current_touch_ids_;
412 // Map of touch ids to their last known location.
413 std::map<int, gfx::PointF> touch_locations_;
415 // The current state.
416 State state_;
418 // A copy of the event from the initial touch press.
419 scoped_ptr<ui::TouchEvent> initial_press_;
421 // Map of touch ids to where its initial press occurred relative to the
422 // screen.
423 std::map<int, gfx::Point> initial_presses_;
425 // In one finger passthrough, the touch is displaced relative to the
426 // last touch exploration location.
427 gfx::Vector2d passthrough_offset_;
429 // Stores the most recent event from a finger that is currently not
430 // sending events through, but might in the future (e.g. before a finger
431 // enters double-tap-hold passthrough, we need to update its location.)
432 scoped_ptr<ui::TouchEvent> last_unused_finger_event_;
434 // The last synthesized mouse move event. When the user double-taps,
435 // we send the passed-through tap to the location of this event.
436 scoped_ptr<ui::TouchEvent> last_touch_exploration_;
438 // A timer that fires after the double-tap delay.
439 base::OneShotTimer<TouchExplorationController> tap_timer_;
441 // A timer that fires to enter passthrough.
442 base::OneShotTimer<TouchExplorationController> passthrough_timer_;
444 // A timer to fire an indicating sound when sliding to change volume.
445 base::RepeatingTimer<TouchExplorationController> sound_timer_;
447 // A default gesture detector config, so we can share the same
448 // timeout and pixel slop constants.
449 ui::GestureDetector::Config gesture_detector_config_;
451 // Gesture Handler to interpret the touch events.
452 scoped_ptr<ui::GestureProviderAura> gesture_provider_;
454 // The previous state entered.
455 State prev_state_;
457 // A copy of the previous event passed.
458 scoped_ptr<ui::TouchEvent> prev_event_;
460 // This toggles whether VLOGS are turned on or not.
461 bool VLOG_on_;
463 // When touch_exploration_controller gets time relative to real time during
464 // testing, this clock is set to the simulated clock and used.
465 base::TickClock* tick_clock_;
467 // Maps the number of fingers in a swipe to the resulting functions that
468 // dispatch key events.
469 std::map<int, base::Closure> left_swipe_gestures_;
470 std::map<int, base::Closure> right_swipe_gestures_;
471 std::map<int, base::Closure> up_swipe_gestures_;
472 std::map<int, base::Closure> down_swipe_gestures_;
474 DISALLOW_COPY_AND_ASSIGN(TouchExplorationController);
477 } // namespace ui
479 #endif // UI_CHROMEOS_TOUCH_EXPLORATION_CONTROLLER_H_