1 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
5 #include "remoting/host/desktop_session_agent.h"
7 #include "base/files/file_util.h"
8 #include "base/logging.h"
9 #include "base/memory/shared_memory.h"
10 #include "ipc/ipc_channel_proxy.h"
11 #include "ipc/ipc_message.h"
12 #include "ipc/ipc_message_macros.h"
13 #include "remoting/base/auto_thread_task_runner.h"
14 #include "remoting/base/constants.h"
15 #include "remoting/host/audio_capturer.h"
16 #include "remoting/host/chromoting_messages.h"
17 #include "remoting/host/desktop_environment.h"
18 #include "remoting/host/input_injector.h"
19 #include "remoting/host/ipc_util.h"
20 #include "remoting/host/remote_input_filter.h"
21 #include "remoting/host/screen_controls.h"
22 #include "remoting/host/screen_resolution.h"
23 #include "remoting/proto/audio.pb.h"
24 #include "remoting/proto/control.pb.h"
25 #include "remoting/proto/event.pb.h"
26 #include "remoting/protocol/clipboard_stub.h"
27 #include "remoting/protocol/input_event_tracker.h"
28 #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
29 #include "third_party/webrtc/modules/desktop_capture/desktop_geometry.h"
30 #include "third_party/webrtc/modules/desktop_capture/mouse_cursor.h"
31 #include "third_party/webrtc/modules/desktop_capture/shared_memory.h"
37 // Routes local clipboard events though the IPC channel to the network process.
38 class DesktopSesssionClipboardStub
: public protocol::ClipboardStub
{
40 explicit DesktopSesssionClipboardStub(
41 scoped_refptr
<DesktopSessionAgent
> desktop_session_agent
);
42 ~DesktopSesssionClipboardStub() override
;
44 // protocol::ClipboardStub implementation.
45 void InjectClipboardEvent(const protocol::ClipboardEvent
& event
) override
;
48 scoped_refptr
<DesktopSessionAgent
> desktop_session_agent_
;
50 DISALLOW_COPY_AND_ASSIGN(DesktopSesssionClipboardStub
);
53 DesktopSesssionClipboardStub::DesktopSesssionClipboardStub(
54 scoped_refptr
<DesktopSessionAgent
> desktop_session_agent
)
55 : desktop_session_agent_(desktop_session_agent
) {
58 DesktopSesssionClipboardStub::~DesktopSesssionClipboardStub() {
61 void DesktopSesssionClipboardStub::InjectClipboardEvent(
62 const protocol::ClipboardEvent
& event
) {
63 desktop_session_agent_
->InjectClipboardEvent(event
);
68 // webrtc::SharedMemory implementation that notifies creating
69 // DesktopSessionAgent when it's deleted.
70 class DesktopSessionAgent::SharedBuffer
: public webrtc::SharedMemory
{
72 static scoped_ptr
<SharedBuffer
> Create(DesktopSessionAgent
* agent
,
75 scoped_ptr
<base::SharedMemory
> memory(new base::SharedMemory());
76 if (!memory
->CreateAndMapAnonymous(size
))
78 return make_scoped_ptr(new SharedBuffer(agent
, memory
.Pass(), size
, id
));
81 ~SharedBuffer() override
{ agent_
->OnSharedBufferDeleted(id()); }
84 SharedBuffer(DesktopSessionAgent
* agent
,
85 scoped_ptr
<base::SharedMemory
> memory
,
88 : SharedMemory(memory
->memory(),
93 base::SharedMemory::GetFdFromSharedMemoryHandle(
98 shared_memory_(memory
.Pass()) {
101 DesktopSessionAgent
* agent_
;
102 scoped_ptr
<base::SharedMemory
> shared_memory_
;
104 DISALLOW_COPY_AND_ASSIGN(SharedBuffer
);
107 DesktopSessionAgent::Delegate::~Delegate() {
110 DesktopSessionAgent::DesktopSessionAgent(
111 scoped_refptr
<AutoThreadTaskRunner
> audio_capture_task_runner
,
112 scoped_refptr
<AutoThreadTaskRunner
> caller_task_runner
,
113 scoped_refptr
<AutoThreadTaskRunner
> input_task_runner
,
114 scoped_refptr
<AutoThreadTaskRunner
> io_task_runner
,
115 scoped_refptr
<AutoThreadTaskRunner
> video_capture_task_runner
)
116 : audio_capture_task_runner_(audio_capture_task_runner
),
117 caller_task_runner_(caller_task_runner
),
118 input_task_runner_(input_task_runner
),
119 io_task_runner_(io_task_runner
),
120 video_capture_task_runner_(video_capture_task_runner
),
121 next_shared_buffer_id_(1),
124 weak_factory_(this) {
125 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
128 bool DesktopSessionAgent::OnMessageReceived(const IPC::Message
& message
) {
129 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
133 IPC_BEGIN_MESSAGE_MAP(DesktopSessionAgent
, message
)
134 IPC_MESSAGE_HANDLER(ChromotingNetworkDesktopMsg_CaptureFrame
,
136 IPC_MESSAGE_HANDLER(ChromotingNetworkDesktopMsg_InjectClipboardEvent
,
137 OnInjectClipboardEvent
)
138 IPC_MESSAGE_HANDLER(ChromotingNetworkDesktopMsg_InjectKeyEvent
,
140 IPC_MESSAGE_HANDLER(ChromotingNetworkDesktopMsg_InjectTextEvent
,
142 IPC_MESSAGE_HANDLER(ChromotingNetworkDesktopMsg_InjectMouseEvent
,
144 IPC_MESSAGE_HANDLER(ChromotingNetworkDesktopMsg_InjectTouchEvent
,
146 IPC_MESSAGE_HANDLER(ChromotingNetworkDesktopMsg_SetScreenResolution
,
148 IPC_MESSAGE_UNHANDLED(handled
= false)
149 IPC_END_MESSAGE_MAP()
151 IPC_BEGIN_MESSAGE_MAP(DesktopSessionAgent
, message
)
152 IPC_MESSAGE_HANDLER(ChromotingNetworkDesktopMsg_StartSessionAgent
,
154 IPC_MESSAGE_UNHANDLED(handled
= false)
155 IPC_END_MESSAGE_MAP()
158 CHECK(handled
) << "Received unexpected IPC type: " << message
.type();
162 void DesktopSessionAgent::OnChannelConnected(int32 peer_pid
) {
163 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
165 VLOG(1) << "IPC: desktop <- network (" << peer_pid
<< ")";
167 desktop_pipe_
.Close();
170 void DesktopSessionAgent::OnChannelError() {
171 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
173 // Make sure the channel is closed.
174 network_channel_
.reset();
175 desktop_pipe_
.Close();
177 // Notify the caller that the channel has been disconnected.
179 delegate_
->OnNetworkProcessDisconnected();
182 webrtc::SharedMemory
* DesktopSessionAgent::CreateSharedMemory(size_t size
) {
183 DCHECK(video_capture_task_runner_
->BelongsToCurrentThread());
185 scoped_ptr
<SharedBuffer
> buffer
=
186 SharedBuffer::Create(this, size
, next_shared_buffer_id_
);
190 // |next_shared_buffer_id_| starts from 1 and incrementing it by 2 makes
191 // sure it is always odd and therefore zero is never used as a valid buffer
194 // It is very unlikely (though theoretically possible) to allocate the same
195 // ID for two different buffers due to integer overflow. It should take
196 // about a year of allocating 100 new buffers every second. Practically
197 // speaking it never happens.
198 next_shared_buffer_id_
+= 2;
200 IPC::PlatformFileForTransit handle
;
202 handle
= buffer
->handle();
204 handle
= base::FileDescriptor(buffer
->handle(), false);
206 SendToNetwork(new ChromotingDesktopNetworkMsg_CreateSharedBuffer(
207 buffer
->id(), handle
, buffer
->size()));
210 return buffer
.release();
213 DesktopSessionAgent::~DesktopSessionAgent() {
214 DCHECK(!audio_capturer_
);
215 DCHECK(!desktop_environment_
);
216 DCHECK(!network_channel_
);
217 DCHECK(!screen_controls_
);
218 DCHECK(!video_capturer_
);
221 const std::string
& DesktopSessionAgent::client_jid() const {
225 void DesktopSessionAgent::DisconnectSession() {
226 SendToNetwork(new ChromotingDesktopNetworkMsg_DisconnectSession());
229 void DesktopSessionAgent::OnLocalMouseMoved(
230 const webrtc::DesktopVector
& new_pos
) {
231 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
233 remote_input_filter_
->LocalMouseMoved(new_pos
);
236 void DesktopSessionAgent::SetDisableInputs(bool disable_inputs
) {
237 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
239 // Do not expect this method to be called because it is only used by It2Me.
243 void DesktopSessionAgent::ResetVideoPipeline() {
244 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
246 // This method is only used by HostExtensionSessions in the network process.
250 void DesktopSessionAgent::OnStartSessionAgent(
251 const std::string
& authenticated_jid
,
252 const ScreenResolution
& resolution
,
253 bool virtual_terminal
) {
254 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
256 DCHECK(!audio_capturer_
);
257 DCHECK(!desktop_environment_
);
258 DCHECK(!input_injector_
);
259 DCHECK(!screen_controls_
);
260 DCHECK(!video_capturer_
);
263 client_jid_
= authenticated_jid
;
265 // Enable the curtain mode.
266 delegate_
->desktop_environment_factory().SetEnableCurtaining(
269 // Create a desktop environment for the new session.
270 desktop_environment_
= delegate_
->desktop_environment_factory().Create(
271 weak_factory_
.GetWeakPtr());
273 // Create the session controller and set the initial screen resolution.
274 screen_controls_
= desktop_environment_
->CreateScreenControls();
275 SetScreenResolution(resolution
);
277 // Create the input injector.
278 input_injector_
= desktop_environment_
->CreateInputInjector();
280 // Hook up the input filter.
281 input_tracker_
.reset(new protocol::InputEventTracker(input_injector_
.get()));
282 remote_input_filter_
.reset(new RemoteInputFilter(input_tracker_
.get()));
285 // LocalInputMonitorWin filters out an echo of the injected input before it
286 // reaches |remote_input_filter_|.
287 remote_input_filter_
->SetExpectLocalEcho(false);
288 #endif // defined(OS_WIN)
290 // Start the input injector.
291 scoped_ptr
<protocol::ClipboardStub
> clipboard_stub(
292 new DesktopSesssionClipboardStub(this));
293 input_injector_
->Start(clipboard_stub
.Pass());
295 // Start the audio capturer.
296 if (delegate_
->desktop_environment_factory().SupportsAudioCapture()) {
297 audio_capturer_
= desktop_environment_
->CreateAudioCapturer();
298 audio_capture_task_runner_
->PostTask(
299 FROM_HERE
, base::Bind(&DesktopSessionAgent::StartAudioCapturer
, this));
302 // Start the video capturer and mouse cursor monitor.
303 video_capturer_
= desktop_environment_
->CreateVideoCapturer();
304 mouse_cursor_monitor_
= desktop_environment_
->CreateMouseCursorMonitor();
305 video_capture_task_runner_
->PostTask(
306 FROM_HERE
, base::Bind(
307 &DesktopSessionAgent::StartVideoCapturerAndMouseMonitor
, this));
310 void DesktopSessionAgent::OnCaptureCompleted(webrtc::DesktopFrame
* frame
) {
311 DCHECK(video_capture_task_runner_
->BelongsToCurrentThread());
313 last_frame_
.reset(frame
);
315 current_size_
= frame
->size();
317 // Serialize webrtc::DesktopFrame.
318 SerializedDesktopFrame serialized_frame
;
319 serialized_frame
.shared_buffer_id
= frame
->shared_memory()->id();
320 serialized_frame
.bytes_per_row
= frame
->stride();
321 serialized_frame
.dimensions
= frame
->size();
322 serialized_frame
.capture_time_ms
= frame
->capture_time_ms();
323 serialized_frame
.dpi
= frame
->dpi();
324 for (webrtc::DesktopRegion::Iterator
i(frame
->updated_region());
325 !i
.IsAtEnd(); i
.Advance()) {
326 serialized_frame
.dirty_region
.push_back(i
.rect());
330 new ChromotingDesktopNetworkMsg_CaptureCompleted(serialized_frame
));
333 void DesktopSessionAgent::OnMouseCursor(webrtc::MouseCursor
* cursor
) {
334 DCHECK(video_capture_task_runner_
->BelongsToCurrentThread());
336 scoped_ptr
<webrtc::MouseCursor
> owned_cursor(cursor
);
339 new ChromotingDesktopNetworkMsg_MouseCursor(*owned_cursor
));
342 void DesktopSessionAgent::OnMouseCursorPosition(
343 webrtc::MouseCursorMonitor::CursorState state
,
344 const webrtc::DesktopVector
& position
) {
345 // We're not subscribing to mouse position changes.
349 void DesktopSessionAgent::InjectClipboardEvent(
350 const protocol::ClipboardEvent
& event
) {
351 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
353 std::string serialized_event
;
354 if (!event
.SerializeToString(&serialized_event
)) {
355 LOG(ERROR
) << "Failed to serialize protocol::ClipboardEvent.";
360 new ChromotingDesktopNetworkMsg_InjectClipboardEvent(serialized_event
));
363 void DesktopSessionAgent::ProcessAudioPacket(scoped_ptr
<AudioPacket
> packet
) {
364 DCHECK(audio_capture_task_runner_
->BelongsToCurrentThread());
366 std::string serialized_packet
;
367 if (!packet
->SerializeToString(&serialized_packet
)) {
368 LOG(ERROR
) << "Failed to serialize AudioPacket.";
372 SendToNetwork(new ChromotingDesktopNetworkMsg_AudioPacket(serialized_packet
));
375 bool DesktopSessionAgent::Start(const base::WeakPtr
<Delegate
>& delegate
,
376 IPC::PlatformFileForTransit
* desktop_pipe_out
) {
377 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
378 DCHECK(delegate_
.get() == nullptr);
380 delegate_
= delegate
;
382 // Create an IPC channel to communicate with the network process.
383 bool result
= CreateConnectedIpcChannel(io_task_runner_
,
387 base::PlatformFile raw_desktop_pipe
= desktop_pipe_
.GetPlatformFile();
389 *desktop_pipe_out
= IPC::PlatformFileForTransit(raw_desktop_pipe
);
390 #elif defined(OS_POSIX)
391 *desktop_pipe_out
= IPC::PlatformFileForTransit(raw_desktop_pipe
, false);
393 #error Unsupported platform.
398 void DesktopSessionAgent::Stop() {
399 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
403 // Make sure the channel is closed.
404 network_channel_
.reset();
409 // Ignore any further callbacks.
410 weak_factory_
.InvalidateWeakPtrs();
413 remote_input_filter_
.reset();
415 // Ensure that any pressed keys or buttons are released.
416 input_tracker_
->ReleaseAll();
417 input_tracker_
.reset();
419 desktop_environment_
.reset();
420 input_injector_
.reset();
421 screen_controls_
.reset();
423 // Stop the audio capturer.
424 audio_capture_task_runner_
->PostTask(
425 FROM_HERE
, base::Bind(&DesktopSessionAgent::StopAudioCapturer
, this));
427 // Stop the video capturer.
428 video_capture_task_runner_
->PostTask(
429 FROM_HERE
, base::Bind(
430 &DesktopSessionAgent::StopVideoCapturerAndMouseMonitor
, this));
434 void DesktopSessionAgent::OnCaptureFrame() {
435 if (!video_capture_task_runner_
->BelongsToCurrentThread()) {
436 video_capture_task_runner_
->PostTask(
438 base::Bind(&DesktopSessionAgent::OnCaptureFrame
, this));
442 mouse_cursor_monitor_
->Capture();
444 // webrtc::DesktopCapturer supports a very few (currently 2) outstanding
445 // capture requests. The requests are serialized on
446 // |video_capture_task_runner()| task runner. If the client issues more
447 // requests, pixel data in captured frames will likely be corrupted but
448 // stability of webrtc::DesktopCapturer will not be affected.
449 video_capturer_
->Capture(webrtc::DesktopRegion());
452 void DesktopSessionAgent::OnInjectClipboardEvent(
453 const std::string
& serialized_event
) {
454 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
456 protocol::ClipboardEvent event
;
457 if (!event
.ParseFromString(serialized_event
)) {
458 LOG(ERROR
) << "Failed to parse protocol::ClipboardEvent.";
462 // InputStub implementations must verify events themselves, so we don't need
463 // verification here. This matches HostEventDispatcher.
464 input_injector_
->InjectClipboardEvent(event
);
467 void DesktopSessionAgent::OnInjectKeyEvent(
468 const std::string
& serialized_event
) {
469 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
471 protocol::KeyEvent event
;
472 if (!event
.ParseFromString(serialized_event
)) {
473 LOG(ERROR
) << "Failed to parse protocol::KeyEvent.";
477 // InputStub implementations must verify events themselves, so we need only
478 // basic verification here. This matches HostEventDispatcher.
479 if (!event
.has_usb_keycode() || !event
.has_pressed()) {
480 LOG(ERROR
) << "Received invalid key event.";
484 remote_input_filter_
->InjectKeyEvent(event
);
487 void DesktopSessionAgent::OnInjectTextEvent(
488 const std::string
& serialized_event
) {
489 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
491 protocol::TextEvent event
;
492 if (!event
.ParseFromString(serialized_event
)) {
493 LOG(ERROR
) << "Failed to parse protocol::TextEvent.";
497 // InputStub implementations must verify events themselves, so we need only
498 // basic verification here. This matches HostEventDispatcher.
499 if (!event
.has_text()) {
500 LOG(ERROR
) << "Received invalid TextEvent.";
504 remote_input_filter_
->InjectTextEvent(event
);
507 void DesktopSessionAgent::OnInjectMouseEvent(
508 const std::string
& serialized_event
) {
509 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
511 protocol::MouseEvent event
;
512 if (!event
.ParseFromString(serialized_event
)) {
513 LOG(ERROR
) << "Failed to parse protocol::MouseEvent.";
517 // InputStub implementations must verify events themselves, so we don't need
518 // verification here. This matches HostEventDispatcher.
519 remote_input_filter_
->InjectMouseEvent(event
);
522 void DesktopSessionAgent::OnInjectTouchEvent(
523 const std::string
& serialized_event
) {
524 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
526 protocol::TouchEvent event
;
527 if (!event
.ParseFromString(serialized_event
)) {
528 LOG(ERROR
) << "Failed to parse protocol::TouchEvent.";
532 remote_input_filter_
->InjectTouchEvent(event
);
535 void DesktopSessionAgent::SetScreenResolution(
536 const ScreenResolution
& resolution
) {
537 DCHECK(caller_task_runner_
->BelongsToCurrentThread());
539 if (screen_controls_
&& resolution
.IsEmpty())
540 screen_controls_
->SetScreenResolution(resolution
);
543 void DesktopSessionAgent::SendToNetwork(IPC::Message
* message
) {
544 if (!caller_task_runner_
->BelongsToCurrentThread()) {
545 caller_task_runner_
->PostTask(
547 base::Bind(&DesktopSessionAgent::SendToNetwork
, this, message
));
551 if (network_channel_
) {
552 network_channel_
->Send(message
);
558 void DesktopSessionAgent::StartAudioCapturer() {
559 DCHECK(audio_capture_task_runner_
->BelongsToCurrentThread());
561 if (audio_capturer_
) {
562 audio_capturer_
->Start(base::Bind(&DesktopSessionAgent::ProcessAudioPacket
,
567 void DesktopSessionAgent::StopAudioCapturer() {
568 DCHECK(audio_capture_task_runner_
->BelongsToCurrentThread());
570 audio_capturer_
.reset();
573 void DesktopSessionAgent::StartVideoCapturerAndMouseMonitor() {
574 DCHECK(video_capture_task_runner_
->BelongsToCurrentThread());
576 if (video_capturer_
) {
577 video_capturer_
->Start(this);
580 if (mouse_cursor_monitor_
) {
581 mouse_cursor_monitor_
->Init(this, webrtc::MouseCursorMonitor::SHAPE_ONLY
);
585 void DesktopSessionAgent::StopVideoCapturerAndMouseMonitor() {
586 DCHECK(video_capture_task_runner_
->BelongsToCurrentThread());
588 video_capturer_
.reset();
590 mouse_cursor_monitor_
.reset();
592 // Video capturer must delete all buffers.
593 DCHECK_EQ(shared_buffers_
, 0);
596 void DesktopSessionAgent::OnSharedBufferDeleted(int id
) {
597 DCHECK(video_capture_task_runner_
->BelongsToCurrentThread());
601 DCHECK_GE(shared_buffers_
, 0);
602 SendToNetwork(new ChromotingDesktopNetworkMsg_ReleaseSharedBuffer(id
));
605 } // namespace remoting