hw:d3d11: rename the Open/Close for each module
[vlc.git] / modules / spu / remoteosd.c
blobd01812c4b1e0e11f1256eaba81e85cfdbf6a8706
1 /*****************************************************************************
2 * remoteosd.c: remote osd over vnc filter module
3 *****************************************************************************
4 * Copyright (C) 2007-2008 Matthias Bauer
5 * $Id$
7 * Authors: Matthias Bauer <matthias dot bauer #_at_# gmx dot ch>
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms of the GNU Lesser General Public License as published by
11 * the Free Software Foundation; either version 2.1 of the License, or
12 * (at your option) any later version.
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implid warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU Lesser General Public License for more details.
19 * You should have received a copy of the GNU Lesser General Public License
20 * along with this program; if not, write to the Free Software Foundation,
21 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
22 *****************************************************************************/
24 /*****************************************************************************
25 * RemoteOSD uses the RFB-Protocol of VNC to display an On-Screen-Display
26 * menu generated by a streaming server as overlay for the streamed video.
28 * The streaming server that implements this is the ffnetdev plugin for VDR.
29 * VDR (VideoDiskRecorder) is an Linux based OpenSource harddisk recorder
30 * software.
31 * The VDR ffnetdev plugin emulates the hardware MPEG decoder and streams the
32 * video over the network instead of hardware video outputs.
33 * The OSD menu of VDR is offered with the RFB protocol to a VNC client.
35 * In fact this video-filter is a simple VNC client that could be also used to
36 * connect to a real VNC host.
37 * Only 8-bit color is supported at the moment.
38 * Using password protected VNC hosts is supported but not recommended, because
39 * you need to insert the used password in the plugin configuration page of
40 * VLC configuration in plain text and it's saved in plain text.
41 *****************************************************************************/
43 //#define VNC_DEBUG
45 /*****************************************************************************
46 * Preamble
47 *****************************************************************************/
49 #ifdef HAVE_CONFIG_H
50 # include "config.h"
51 #endif
53 #include <vlc_common.h>
54 #include <vlc_plugin.h>
55 #include <vlc_filter.h>
56 #include <vlc_mouse.h>
57 #include <vlc_subpicture.h>
58 #include <vlc_actions.h> /* KEY_MODIFIER_CTRL */
60 #include <vlc_network.h> /* net_*, htonl */
61 #include <gcrypt.h> /* to encrypt password */
62 #include <vlc_gcrypt.h>
64 #include "remoteosd_rfbproto.h" /* type definitions of the RFB protocol for VNC */
66 /*****************************************************************************
67 * Module descriptor
68 *****************************************************************************/
69 #define READ_BUFFER_SIZE 1000000
71 #define RMTOSD_HOST_TEXT N_("VNC Host")
72 #define RMTOSD_HOST_LONGTEXT N_( \
73 "VNC hostname or IP address." )
75 #define RMTOSD_PORT_TEXT N_("VNC Port")
76 #define RMTOSD_PORT_LONGTEXT N_( \
77 "VNC port number." )
79 #define RMTOSD_PASSWORD_TEXT N_("VNC Password")
80 #define RMTOSD_PASSWORD_LONGTEXT N_( \
81 "VNC password." )
83 #define RMTOSD_UPDATE_TEXT N_("VNC poll interval" )
84 #define RMTOSD_UPDATE_LONGTEXT N_( \
85 "In this interval an update from VNC is requested, default every 300 ms.")
87 #define RMTOSD_POLL_TEXT N_("VNC polling")
88 #define RMTOSD_POLL_LONGTEXT N_( \
89 "Activate VNC polling. Do NOT activate for use as VDR ffnetdev client." )
91 #define RMTOSD_MOUSE_TEXT N_("Mouse events")
92 #define RMTOSD_MOUSE_LONGTEXT N_( \
93 "Send mouse events to VNC host. Not needed for use as VDR ffnetdev client." )
95 #define RMTOSD_KEYS_TEXT N_("Key events")
96 #define RMTOSD_KEYS_LONGTEXT N_( \
97 "Send key events to VNC host." )
99 #define RMTOSD_ALPHA_TEXT N_("Alpha transparency value (default 255)")
100 #define RMTOSD_ALPHA_LONGTEXT N_( \
101 "The transparency of the OSD VNC can be changed by giving a value " \
102 "between 0 and 255. A lower value specifies more transparency a higher " \
103 "means less transparency. The default is being not transparent " \
104 "(value 255) the minimum is fully transparent (value 0)." )
106 #define RMTOSD_CFG "rmtosd-"
108 #define RMTOSD_UPDATE_MIN 200
109 #define RMTOSD_UPDATE_DEFAULT 1000
110 #define RMTOSD_UPDATE_MAX 300
112 static int CreateFilter ( vlc_object_t * );
113 static void DestroyFilter( vlc_object_t * );
115 vlc_module_begin ()
116 set_description( N_("Remote-OSD over VNC") )
117 set_capability( "sub source", 100 )
118 set_shortname( N_("Remote-OSD") )
119 set_category( CAT_VIDEO )
120 set_subcategory( SUBCAT_VIDEO_SUBPIC )
121 add_shortcut( "rmtosd" )
122 set_callbacks( CreateFilter, DestroyFilter )
124 add_string( RMTOSD_CFG "host", "myvdr", RMTOSD_HOST_TEXT,
125 RMTOSD_HOST_LONGTEXT, false )
126 add_integer_with_range( RMTOSD_CFG "port", 20001, 1, 0xFFFF,
127 RMTOSD_PORT_TEXT, RMTOSD_PORT_LONGTEXT, false )
128 add_password( RMTOSD_CFG "password", "", RMTOSD_PASSWORD_TEXT,
129 RMTOSD_PASSWORD_LONGTEXT, false )
130 add_integer_with_range( RMTOSD_CFG "update", RMTOSD_UPDATE_DEFAULT,
131 RMTOSD_UPDATE_MIN, RMTOSD_UPDATE_MAX, RMTOSD_UPDATE_TEXT,
132 RMTOSD_UPDATE_LONGTEXT, true )
133 add_bool( RMTOSD_CFG "vnc-polling", false,
134 RMTOSD_POLL_TEXT , RMTOSD_POLL_LONGTEXT, false )
135 add_bool( RMTOSD_CFG "mouse-events", false,
136 RMTOSD_MOUSE_TEXT , RMTOSD_MOUSE_LONGTEXT, false )
137 add_bool( RMTOSD_CFG "key-events", false,
138 RMTOSD_KEYS_TEXT , RMTOSD_KEYS_LONGTEXT, false )
139 add_integer_with_range( RMTOSD_CFG "alpha", 255, 0, 255,
140 RMTOSD_ALPHA_TEXT, RMTOSD_ALPHA_LONGTEXT, true )
142 vlc_module_end ()
145 /*****************************************************************************
146 * Local prototypes
147 *****************************************************************************/
148 #define CHALLENGESIZE 16
149 #define MAX_VNC_SERVER_NAME_LENGTH 255
151 /* subsource functions */
152 static subpicture_t *Filter( filter_t *, mtime_t );
154 static int MouseEvent( filter_t *,
155 const vlc_mouse_t *,
156 const vlc_mouse_t *,
157 const video_format_t * );
159 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
160 vlc_value_t oldval, vlc_value_t newval, void *p_data );
162 static void* vnc_worker_thread ( void * );
164 static void* update_request_thread( void * );
166 static bool process_server_message ( filter_t *p_filter,
167 rfbServerToClientMsg *msg );
169 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
170 int r, int g, int b );
172 static inline bool fill_rect( filter_sys_t* p_sys,
173 uint16_t i_x, uint16_t i_y,
174 uint16_t i_w, uint16_t i_h,
175 uint8_t i_color );
176 static inline bool copy_rect( filter_sys_t* p_sys,
177 uint16_t i_x, uint16_t i_y,
178 uint16_t i_w, uint16_t i_h,
179 uint16_t i_sx, uint16_t i_sy );
182 static inline bool raw_line( filter_sys_t* p_sys,
183 uint16_t i_x, uint16_t i_y,
184 uint16_t i_w );
186 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd );
189 /*****************************************************************************
190 * Sub source code
191 *****************************************************************************/
193 /*****************************************************************************
194 * Local prototypes
195 *****************************************************************************/
196 struct filter_sys_t
198 vlc_mutex_t lock; /* To lock for read/write on picture */
200 bool b_need_update; /* VNC picture is updated, do update the OSD*/
201 uint8_t i_alpha; /* alpha transparency value */
203 char *psz_host; /* VNC host */
205 char *psz_passwd; /* VNC password */
207 picture_t *p_pic; /* The picture with OSD data from VNC */
209 int i_socket; /* Socket used for VNC */
211 uint16_t i_vnc_width; /* The with of the VNC screen */
212 uint16_t i_vnc_height; /* The height of the VNC screen */
214 bool b_vnc_key_events; /* Send KeyEvents ? */
215 bool b_alpha_from_vnc; /* Special ffnetdev alpha feature enabled ? */
217 char read_buffer[READ_BUFFER_SIZE];
219 vlc_thread_t worker_thread;
221 uint8_t ar_color_table_yuv[256][4];
224 /*****************************************************************************
225 * CreateFilter: Create the filter and open the definition file
226 *****************************************************************************/
227 static int CreateFilter ( vlc_object_t *p_this )
229 filter_t *p_filter = (filter_t *)p_this;
231 filter_sys_t *p_sys = malloc( sizeof (*p_sys) );
232 if( unlikely(p_sys == NULL) )
233 return VLC_ENOMEM;
235 /* Populating struct */
236 vlc_mutex_init( &p_sys->lock );
237 p_sys->b_need_update = false;
238 p_sys->psz_host = var_InheritString( p_this, RMTOSD_CFG "host" );
239 p_sys->psz_passwd = var_InheritString( p_this, RMTOSD_CFG "password" );
240 p_sys->i_alpha = var_InheritInteger( p_this, RMTOSD_CFG "alpha" );
241 p_sys->p_pic = NULL;
242 p_sys->i_socket = -1;
244 memset( p_sys->ar_color_table_yuv, 255,
245 sizeof( p_sys->ar_color_table_yuv ) );
247 if( p_sys->psz_host == NULL )
249 msg_Err( p_filter, "unable to get vnc host" );
250 goto error;
253 if( p_sys->psz_passwd == NULL )
255 msg_Err( p_filter, "unable to get vnc password" );
256 goto error;
259 p_filter->p_sys = p_sys;
261 vlc_gcrypt_init();
263 /* create the vnc worker thread */
264 if( vlc_clone( &p_sys->worker_thread,
265 vnc_worker_thread, p_filter, VLC_THREAD_PRIORITY_LOW ) )
267 msg_Err( p_filter, "cannot spawn vnc message reader thread" );
268 goto error;
271 /* Attach subpicture source callback */
272 p_filter->pf_sub_source = Filter;
274 es_format_Init( &p_filter->fmt_out, SPU_ES, VLC_CODEC_SPU );
275 p_filter->fmt_out.i_priority = ES_PRIORITY_SELECTABLE_MIN;
277 if( var_InheritBool( p_this, RMTOSD_CFG "mouse-events" ) )
278 p_filter->pf_sub_mouse = MouseEvent;
280 p_sys->b_vnc_key_events = var_InheritBool( p_this,
281 RMTOSD_CFG "key-events" );
282 if( p_sys->b_vnc_key_events )
283 var_AddCallback( p_filter->obj.libvlc, "key-pressed", KeyEvent, p_this );
285 msg_Dbg( p_filter, "osdvnc filter started" );
287 return VLC_SUCCESS;
289 error:
290 msg_Err( p_filter, "osdvnc filter discarded" );
292 vlc_mutex_destroy( &p_sys->lock );
293 free( p_sys->psz_host );
294 free( p_sys->psz_passwd );
295 free( p_sys );
297 return VLC_EGENERIC;
300 /*****************************************************************************
301 * DestroyFilter: Make a clean exit of this plugin
302 *****************************************************************************/
303 static void DestroyFilter( vlc_object_t *p_this )
305 filter_t *p_filter = (filter_t*)p_this;
306 filter_sys_t *p_sys = p_filter->p_sys;
308 msg_Dbg( p_filter, "DestroyFilter called." );
310 if( p_sys->b_vnc_key_events )
311 var_DelCallback( p_filter->obj.libvlc, "key-pressed", KeyEvent, p_this );
313 vlc_cancel( p_sys->worker_thread );
314 vlc_join( p_sys->worker_thread, NULL );
316 if( p_sys->p_pic != NULL )
317 picture_Release( p_sys->p_pic );
318 if( p_sys->i_socket >= 0 )
319 net_Close( p_sys->i_socket );
321 vlc_mutex_destroy( &p_sys->lock );
322 free( p_sys->psz_host );
323 free( p_sys->psz_passwd );
324 free( p_sys );
327 static bool read_exact( filter_t *obj, int fd, void *buf, size_t len )
329 return (ssize_t)len == net_Read( obj, fd, buf, len );
333 static bool write_exact( filter_t *obj, int fd, const void *buf, size_t len )
335 return (ssize_t)len == net_Write( obj, fd, buf, len );
338 static int vnc_connect( filter_t *p_filter )
340 filter_sys_t *p_sys = p_filter->p_sys;
342 int port = var_InheritInteger( p_filter, RMTOSD_CFG "port" );
344 int fd = net_ConnectTCP( p_filter, p_sys->psz_host, port );
345 if( fd == -1 )
347 msg_Err( p_filter, "Could not connect to VNC host" );
348 return -1;
351 msg_Dbg( p_filter, "Reading protocol version" );
353 rfbProtocolVersionMsg pv;
354 if ( !read_exact( p_filter, fd, pv, sz_rfbProtocolVersionMsg ) )
356 msg_Err( p_filter, "Could not read version message" );
357 goto error;
359 pv[sz_rfbProtocolVersionMsg] = '\0'; /* pv size is sz_rfbProtocolVersionMsg+1 */
361 msg_Dbg( p_filter, "Server version is %s", pv );
363 strncpy(pv, "RFB 003.003\n", sz_rfbProtocolVersionMsg);
365 if( !write_exact(p_filter, fd, pv, sz_rfbProtocolVersionMsg) )
367 msg_Err( p_filter, "Could not write version message" );
368 goto error;
371 msg_Dbg( p_filter, "Reading authentication scheme" );
372 uint32_t i_authScheme;
373 if( !read_exact( p_filter, fd, &i_authScheme, 4 ) )
375 msg_Err( p_filter, "Could not read authentication scheme" );
376 goto error;
378 i_authScheme = htonl(i_authScheme);
380 msg_Dbg( p_filter, "Authentication scheme = %x", i_authScheme );
381 if ( i_authScheme == rfbConnFailed )
383 msg_Err( p_filter, "Connection rejected by server" );
384 goto error;
386 if (i_authScheme == rfbVncAuth)
388 unsigned char challenge[CHALLENGESIZE];
389 if ( !read_exact( p_filter, fd, challenge, CHALLENGESIZE ) )
391 msg_Err( p_filter, "Could not read password challenge" );
392 goto error;
395 vnc_encrypt_bytes( challenge, p_sys->psz_passwd );
397 if( !write_exact(p_filter, fd, challenge, CHALLENGESIZE ) )
399 msg_Err( p_filter, "Could not write password" );
400 goto error;
402 uint32_t i_authResult;
403 if( !read_exact( p_filter, fd, &i_authResult, 4 ) )
405 msg_Err( p_filter, "Could not read authentication result" );
406 goto error;
408 i_authResult = htonl(i_authResult);
409 if (i_authResult != rfbVncAuthOK)
411 msg_Err( p_filter, "VNC authentication failed" );
412 goto error;
416 msg_Dbg( p_filter, "Writing client init message" );
417 rfbClientInitMsg ci;
418 ci.shared = 1;
419 if( !write_exact( p_filter, fd, &ci, sz_rfbClientInitMsg ) )
421 msg_Err( p_filter, "Could not write client init message" );
422 goto error;
425 msg_Dbg( p_filter, "Reading server init message" );
426 rfbServerInitMsg si;
427 if( !read_exact( p_filter, fd, &si, sz_rfbServerInitMsg ) )
429 msg_Err( p_filter, "Could not read server init message" );
430 goto error;
432 si.framebufferWidth = htons(si.framebufferWidth);
433 si.framebufferHeight = htons(si.framebufferHeight);
434 si.format.redMax = htons(si.format.redMax);
435 si.format.greenMax = htons(si.format.greenMax);
436 si.format.blueMax = htons(si.format.blueMax);
438 p_sys->i_vnc_width = si.framebufferWidth;
439 p_sys->i_vnc_height = si.framebufferHeight;
441 msg_Dbg( p_filter, "Servers preferred pixelformat: "
442 "%ux%u, R(%u),G(%u),B(%u), %u bit, depht=%u, %s",
443 si.framebufferWidth,
444 si.framebufferHeight,
445 si.format.redMax,
446 si.format.greenMax,
447 si.format.blueMax,
448 si.format.bitsPerPixel,
449 si.format.depth,
450 si.format.trueColour ? "TrueColor" : "Not-TrueColor");
452 uint32_t i_nameLength = htonl(si.nameLength);
453 if( i_nameLength > MAX_VNC_SERVER_NAME_LENGTH )
455 msg_Err( p_filter, "Server name too long" );
456 goto error;
458 char s_ServerName[MAX_VNC_SERVER_NAME_LENGTH+1];
460 msg_Dbg( p_filter, "Reading server name with size = %u", i_nameLength );
461 if( !read_exact( p_filter, fd, s_ServerName, i_nameLength ) )
463 msg_Err( p_filter, "Could not read server name" );
464 goto error;
466 s_ServerName[i_nameLength] = '\0';
468 if( strcmp( s_ServerName, "VDR-OSD") == 0 )
470 msg_Dbg( p_filter, "Server is a VDR" );
471 p_sys->b_alpha_from_vnc = true;
473 else
475 msg_Dbg( p_filter, "Server is a normal VNC" );
476 p_sys->b_alpha_from_vnc = false;
480 msg_Dbg( p_filter, "Server init message read properly" );
481 msg_Dbg( p_filter, "Server name is %s", s_ServerName );
483 msg_Dbg( p_filter, "Writing SetPixelFormat message" );
485 rfbSetPixelFormatMsg sp;
486 sp.type = rfbSetPixelFormat;
487 sp.pad1 = sp.pad2 = 0;
488 sp.format.bitsPerPixel = 8;
489 sp.format.depth = 8 ;
490 sp.format.bigEndian = 1;
491 sp.format.trueColour = 0;
492 sp.format.redMax = htons(31);
493 sp.format.greenMax = htons(31);
494 sp.format.blueMax = htons(31);
495 sp.format.redShift = 10;
496 sp.format.greenShift = 5;
497 sp.format.blueShift = 0;
498 sp.format.pad1 = sp.format.pad2 = 0;
500 if( !write_exact( p_filter, fd, &sp, sz_rfbSetPixelFormatMsg) )
502 msg_Err( p_filter, "Could not write SetPixelFormat message" );
503 goto error;
506 msg_Dbg( p_filter, "Writing SetEncodings message" );
508 rfbSetEncodingsMsg se;
509 se.type = rfbSetEncodings;
510 se.pad = 0;
511 se.nEncodings = htons( p_sys->b_alpha_from_vnc ? 3 : 2 );
513 if( !write_exact( p_filter, fd, &se, sz_rfbSetEncodingsMsg) )
515 msg_Err( p_filter, "Could not write SetEncodings message begin" );
516 goto error;
519 uint32_t i_encoding;
521 msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingCopyRect" );
522 i_encoding = htonl(rfbEncodingCopyRect);
523 if( !write_exact( p_filter, fd, &i_encoding, 4) )
525 msg_Err( p_filter, "Could not write encoding type rfbEncodingCopyRect." );
526 goto error;
529 msg_Dbg( p_filter, "Writing SetEncodings rfbEncodingRRE" );
530 i_encoding = htonl(rfbEncodingRRE);
531 if( !write_exact(p_filter, fd, &i_encoding, 4) )
533 msg_Err( p_filter, "Could not write encoding type rfbEncodingRRE." );
534 goto error;
537 if( p_sys->b_alpha_from_vnc )
539 msg_Dbg( p_filter, "Writing SetEncodings rfbEncSpecialUseAlpha" );
540 i_encoding = 0x00F0FFFF; /* rfbEncSpecialUseAlpha is 0xFFFFF000
541 * before we swap it */
542 if( !write_exact(p_filter, fd, &i_encoding, 4) )
544 msg_Err( p_filter, "Could not write encoding type rfbEncSpecialUseAlpha." );
545 goto error;
549 return fd;
550 error:
551 net_Close( fd );
552 return -1;
555 static int write_update_request(filter_t *p_filter, bool incremental)
557 filter_sys_t *p_sys = p_filter->p_sys;
558 rfbFramebufferUpdateRequestMsg udr;
560 udr.type = rfbFramebufferUpdateRequest;
561 udr.incremental = incremental;
562 udr.x = 0;
563 udr.y = 0;
564 udr.w = htons(p_sys->i_vnc_width);
565 udr.h = htons(p_sys->i_vnc_height);
567 int w = write_exact(p_filter, p_sys->i_socket, &udr,
568 sz_rfbFramebufferUpdateRequestMsg);
569 if( !w )
570 msg_Err( p_filter, "Could not write rfbFramebufferUpdateRequestMsg." );
571 return w;
574 static void update_thread_cleanup( void *p )
576 vlc_thread_t *th = p;
578 vlc_cancel( *th );
579 vlc_join( *th, NULL );
582 static void dummy_cleanup( void *p )
584 (void) p;
587 static void* vnc_worker_thread( void *obj )
589 filter_t* p_filter = (filter_t*)obj;
590 filter_sys_t *p_sys = p_filter->p_sys;
591 vlc_thread_t update_thread;
592 int canc = vlc_savecancel ();
594 msg_Dbg( p_filter, "VNC worker thread started" );
596 int fd = vnc_connect( p_filter );
597 if( fd == -1 )
599 msg_Err( p_filter, "Error occurred while handshaking VNC host" );
600 return NULL;
603 /* Create an empty picture for VNC the data */
604 picture_t *pic = picture_New( VLC_CODEC_YUVA, p_sys->i_vnc_width,
605 p_sys->i_vnc_height, 1, 1 );
606 if( likely(pic != NULL) )
608 vlc_mutex_lock( &p_sys->lock );
609 p_sys->i_socket = fd;
610 p_sys->p_pic = pic;
611 vlc_mutex_unlock( &p_sys->lock );
613 else
615 net_Close( fd );
616 return NULL;
619 write_update_request( p_filter, false );
621 /* create the update request thread */
622 bool polling = var_InheritBool( p_filter, RMTOSD_CFG "vnc-polling" );
623 if( polling
624 && vlc_clone( &update_thread, update_request_thread,
625 p_filter, VLC_THREAD_PRIORITY_LOW ) )
627 msg_Err( p_filter, "cannot spawn VNC update request thread" );
628 polling = false;
631 vlc_cleanup_push( polling ? update_thread_cleanup : dummy_cleanup,
632 &update_thread );
634 /* connection is initialized, now read and handle server messages */
635 for( ;; )
637 rfbServerToClientMsg msg;
638 int i_msgSize;
640 memset( &msg, 0, sizeof(msg) );
641 vlc_restorecancel (canc);
643 if( !read_exact(p_filter, fd, &msg, 1 ) )
645 msg_Err( p_filter, "Error while waiting for next server message");
646 break;
648 switch (msg.type)
650 case rfbFramebufferUpdate:
651 i_msgSize = sz_rfbFramebufferUpdateMsg;
652 break;
653 case rfbSetColourMapEntries:
654 i_msgSize = sz_rfbSetColourMapEntriesMsg;
655 break;
656 case rfbBell:
657 i_msgSize = sz_rfbBellMsg;
658 break;
659 case rfbServerCutText:
660 i_msgSize = sz_rfbServerCutTextMsg;
661 break;
662 case rfbReSizeFrameBuffer:
663 i_msgSize = sz_rfbReSizeFrameBufferMsg;
664 break;
665 default:
666 i_msgSize = 0;
667 msg_Err( p_filter, "Invalid message %u received", msg.type );
668 break;
671 if( i_msgSize <= 0 )
672 break;
674 if( --i_msgSize > 0 )
676 if ( !read_exact( p_filter, fd, ((char *)&msg) + 1, i_msgSize ) )
678 msg_Err( p_filter, "Error while reading message of type %u",
679 msg.type );
680 break;
684 canc = vlc_savecancel ();
685 process_server_message( p_filter, &msg);
688 vlc_cleanup_pop();
689 if( polling )
690 update_thread_cleanup( &update_thread );
692 msg_Dbg( p_filter, "VNC message reader thread ended" );
693 vlc_restorecancel (canc);
694 return NULL;
697 static void* update_request_thread( void *obj )
699 filter_t* p_filter = (filter_t*)obj;
700 int canc = vlc_savecancel();
701 mtime_t interval = var_InheritInteger( p_filter, RMTOSD_CFG "update" );
702 vlc_restorecancel(canc);
704 if( interval < 100 )
705 interval = 100;
706 interval *= 1000; /* ms -> µs */
709 msleep( interval );
710 while( write_update_request( p_filter, true ) );
712 return NULL;
715 static bool process_server_message ( filter_t *p_filter,
716 rfbServerToClientMsg *msg )
718 filter_sys_t *p_sys = p_filter->p_sys;
720 switch (msg->type)
722 case rfbFramebufferUpdate:
724 msg->fu.nRects = htons(msg->fu.nRects);
725 rfbFramebufferUpdateRectHeader hdr;
727 for (int i_rect = 0; i_rect < msg->fu.nRects; i_rect++)
729 if (!read_exact(p_filter, p_sys->i_socket, &hdr,
730 sz_rfbFramebufferUpdateRectHeader ) )
732 msg_Err( p_filter, "Could not read FrameBufferUpdate header" );
733 return false;
735 hdr.r.x = htons(hdr.r.x);
736 hdr.r.y = htons(hdr.r.y);
737 hdr.r.w = htons(hdr.r.w);
738 hdr.r.h = htons(hdr.r.h);
739 hdr.encoding = htonl(hdr.encoding);
741 switch (hdr.encoding)
743 case rfbEncodingRaw:
745 int i_line;
746 for (i_line = 0; i_line < hdr.r.h; i_line++)
748 if ( !read_exact( p_filter, p_sys->i_socket,
749 p_sys->read_buffer, hdr.r.w ) )
751 msg_Err( p_filter,
752 "Could not read FrameBufferUpdate line data" );
753 return false;
755 vlc_mutex_lock( &p_sys->lock );
756 if ( !raw_line( p_sys, hdr.r.x,
757 hdr.r.y + i_line,
758 hdr.r.w ) )
760 msg_Err( p_filter, "raw_line failed." );
761 vlc_mutex_unlock( &p_sys->lock );
762 return false;
764 vlc_mutex_unlock( &p_sys->lock );
767 break;
769 case rfbEncodingCopyRect:
771 rfbCopyRect rect;
773 if ( !read_exact( p_filter, p_sys->i_socket,
774 &rect, sz_rfbCopyRect ) )
776 msg_Err( p_filter, "Could not read rfbCopyRect" );
777 return false;
779 rect.srcX = htons( rect.srcX );
780 rect.srcY = htons( rect.srcY );
782 vlc_mutex_lock( &p_sys->lock );
783 if ( !copy_rect( p_sys,
784 hdr.r.x, hdr.r.y,
785 hdr.r.w, hdr.r.h,
786 rect.srcX, rect.srcY ) )
788 msg_Err( p_filter, "copy_rect failed." );
789 vlc_mutex_unlock( &p_sys->lock );
790 return false;
792 vlc_mutex_unlock( &p_sys->lock );
794 break;
796 case rfbEncodingRRE:
798 rfbRREHeader rrehdr;
799 if ( !read_exact( p_filter, p_sys->i_socket,
800 &rrehdr, sz_rfbRREHeader ) )
802 msg_Err( p_filter, "Could not read rfbRREHeader" );
803 return false;
805 uint8_t i_pixcolor;
806 if ( !read_exact( p_filter, p_sys->i_socket,
807 &i_pixcolor, 1 ) )
809 msg_Err( p_filter, "Could not read RRE pixcolor" );
810 return false;
813 vlc_mutex_lock( &p_sys->lock );
814 if ( !fill_rect( p_sys,
815 hdr.r.x, hdr.r.y,
816 hdr.r.w, hdr.r.h,
817 i_pixcolor) )
819 msg_Err( p_filter, "main fill_rect failed." );
820 vlc_mutex_unlock( &p_sys->lock );
821 return false;
823 vlc_mutex_unlock( &p_sys->lock );
825 rrehdr.nSubrects = htonl(rrehdr.nSubrects);
827 int i_datasize = rrehdr.nSubrects *
828 ( sizeof(i_pixcolor) + sz_rfbRectangle ) ;
829 if ( i_datasize > READ_BUFFER_SIZE )
831 msg_Err( p_filter, "Buffer too small, "
832 "need %u bytes", i_datasize );
833 return false;
835 if ( !read_exact( p_filter, p_sys->i_socket,
836 p_sys->read_buffer, i_datasize ) )
838 msg_Err( p_filter,
839 "Could not read RRE subrect data" );
840 return false;
843 uint32_t i_subrect;
844 rfbRectangle* p_subrect;
845 int i_offset = 0;
846 vlc_mutex_lock( &p_sys->lock );
847 for ( i_subrect = 0;
848 i_subrect < rrehdr.nSubrects; i_subrect++)
850 i_pixcolor = p_sys->read_buffer[i_offset];
851 i_offset += sizeof(i_pixcolor);
852 p_subrect =
853 (rfbRectangle*)(p_sys->read_buffer + i_offset);
854 i_offset += sz_rfbRectangle;
856 if (!fill_rect( p_sys,
857 htons(p_subrect->x) + hdr.r.x,
858 htons(p_subrect->y) + hdr.r.y,
859 htons(p_subrect->w),
860 htons(p_subrect->h),
861 i_pixcolor) )
863 msg_Err( p_filter,
864 "subrect %u fill_rect failed.", i_subrect );
865 vlc_mutex_unlock( &p_sys->lock );
866 return false;
869 vlc_mutex_unlock( &p_sys->lock );
871 break;
876 vlc_mutex_lock( &p_sys->lock );
877 p_sys->b_need_update = true;
878 vlc_mutex_unlock( &p_sys->lock );
880 return true;
882 case rfbSetColourMapEntries:
884 msg->scme.nColours = htons(msg->scme.nColours);
885 msg->scme.firstColour = htons(msg->scme.firstColour);
886 int i_datasize;
887 if ( p_sys->b_alpha_from_vnc )
889 i_datasize = 2 * msg->scme.nColours * 4;
891 else
893 i_datasize = 2 * msg->scme.nColours * 3;
895 if ( i_datasize > READ_BUFFER_SIZE )
897 msg_Err( p_filter, "Buffer too small, need %u bytes",
898 i_datasize );
899 return false;
902 if ( !read_exact( p_filter, p_sys->i_socket,
903 p_sys->read_buffer, i_datasize ) )
905 msg_Err( p_filter, "Could not read color map data" );
906 return false;
909 uint8_t i_red, i_green, i_blue, i_alpha, i_color_index;
910 uint16_t i_offset = 0;
911 i_alpha = 255;
913 for (int i = 0; i < msg->scme.nColours; i++)
915 i_color_index = i+msg->scme.firstColour;
916 if ( p_sys->b_alpha_from_vnc )
918 i_alpha = p_sys->read_buffer[i_offset];
919 i_offset += 2;
921 i_red = p_sys->read_buffer[i_offset];
922 i_offset += 2;
923 i_green = p_sys->read_buffer[i_offset];
924 i_offset += 2;
925 i_blue = p_sys->read_buffer[i_offset];
926 i_offset += 2;
927 rgb_to_yuv( &p_sys->ar_color_table_yuv[i_color_index][0],
928 &p_sys->ar_color_table_yuv[i_color_index][1],
929 &p_sys->ar_color_table_yuv[i_color_index][2],
930 i_red,
931 i_green,
932 i_blue );
933 p_sys->ar_color_table_yuv[i][3] = i_alpha;
936 return true;
938 case rfbBell:
939 msg_Err( p_filter, "rfbBell received" );
940 return true;
942 case rfbServerCutText:
943 msg->sct.length = htons(msg->sct.length);
944 if ( msg->sct.length > READ_BUFFER_SIZE )
946 msg_Err( p_filter, "Buffer too small, need %u bytes", msg->sct.length );
947 return false;
949 if ( !read_exact( p_filter, p_sys->i_socket,
950 p_sys->read_buffer, msg->sct.length ) )
952 msg_Err( p_filter, "Could not read Reading rfbServerCutText data" );
953 return false;
955 return true;
957 case rfbReSizeFrameBuffer:
958 msg_Err( p_filter, "Reading rfbReSizeFrameBuffer not implemented" );
959 return false;
961 default:
962 msg_Err( p_filter, "Invalid message %u received", msg->type );
963 return false;
965 return false;
968 /****************************************************************************
969 * Filter: the whole thing
970 ****************************************************************************
971 * This function outputs subpictures at regular time intervals.
972 ****************************************************************************/
973 static subpicture_t *Filter( filter_t *p_filter, mtime_t date )
975 filter_sys_t *p_sys = p_filter->p_sys;
976 subpicture_t *p_spu;
977 subpicture_region_t *p_region;
978 video_format_t fmt;
979 picture_t *p_pic = NULL;
981 vlc_mutex_lock( &p_sys->lock );
983 if( p_sys->b_need_update )
984 p_pic = p_sys->p_pic;
986 if( p_pic == NULL )
988 vlc_mutex_unlock( &p_sys->lock );
989 return NULL;
992 /* Allocate the subpicture internal data. */
993 p_spu = filter_NewSubpicture( p_filter );
994 if( !p_spu )
996 vlc_mutex_unlock( &p_sys->lock );
997 return NULL;
1000 p_spu->b_absolute = false;
1001 p_spu->i_start = date;
1002 p_spu->i_stop = 0;
1003 p_spu->b_ephemer = true;
1005 /* Create new SPU region */
1006 memset( &fmt, 0, sizeof(video_format_t) );
1007 fmt.i_chroma = VLC_CODEC_YUVA;
1008 fmt.i_sar_num = fmt.i_sar_den = 1;
1009 fmt.i_width = fmt.i_visible_width = p_pic->p[Y_PLANE].i_visible_pitch;
1010 fmt.i_height = fmt.i_visible_height = p_pic->p[Y_PLANE].i_visible_lines;
1011 fmt.i_x_offset = fmt.i_y_offset = 0;
1012 p_region = subpicture_region_New( &fmt );
1013 if( !p_region )
1015 msg_Err( p_filter, "cannot allocate SPU region" );
1016 subpicture_Delete( p_spu );
1017 vlc_mutex_unlock( &p_sys->lock );
1018 return NULL;
1021 /* FIXME the copy is probably not needed anymore */
1022 picture_Copy( p_region->p_picture, p_pic );
1024 p_sys->b_need_update = false;
1026 vlc_mutex_unlock( &p_sys->lock );
1028 /* set to one of the 9 relative locations */
1029 p_region->i_align = 0; /* Center */
1030 p_spu->b_absolute = false;
1033 p_spu->i_original_picture_width = 0; /*Let vout core do the horizontal scaling */
1034 p_spu->i_original_picture_height = fmt.i_height;
1036 p_spu->p_region = p_region;
1038 p_spu->i_alpha = ( p_sys->i_alpha );
1040 return p_spu;
1044 static inline void rgb_to_yuv( uint8_t *y, uint8_t *u, uint8_t *v,
1045 int r, int g, int b )
1047 *y = ( ( ( 66 * r + 129 * g + 25 * b + 128 ) >> 8 ) + 16 );
1048 *u = ( ( -38 * r - 74 * g + 112 * b + 128 ) >> 8 ) + 128 ;
1049 *v = ( ( 112 * r - 94 * g - 18 * b + 128 ) >> 8 ) + 128 ;
1052 static inline bool fill_rect( filter_sys_t* p_sys,
1053 uint16_t i_x, uint16_t i_y,
1054 uint16_t i_w, uint16_t i_h,
1055 uint8_t i_color)
1057 plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1058 plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1059 plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1060 plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1061 int i_pitch = p_outY->i_pitch;
1062 int i_lines = p_outY->i_lines;
1063 if ( i_x + i_w > i_pitch)
1064 return false;
1065 if ( i_y + i_h > i_lines)
1066 return false;
1067 int i_line_offset = i_y * i_pitch;
1068 uint8_t i_yuv_y = p_sys->ar_color_table_yuv[i_color][0];
1069 uint8_t i_yuv_u = p_sys->ar_color_table_yuv[i_color][1];
1070 uint8_t i_yuv_v = p_sys->ar_color_table_yuv[i_color][2];
1071 uint8_t i_alpha = p_sys->ar_color_table_yuv[i_color][3];
1072 for( int i_line = 0; i_line < i_h; i_line++ )
1074 for( int i_column = 0; i_column < i_w; i_column++ )
1076 int i_total_offset = i_line_offset + i_x + i_column;
1077 p_outY->p_pixels[ i_total_offset ] = i_yuv_y;
1078 p_outU->p_pixels[ i_total_offset ] = i_yuv_u;
1079 p_outV->p_pixels[ i_total_offset ] = i_yuv_v;
1080 p_outA->p_pixels[ i_total_offset ] = i_alpha;
1082 i_line_offset += i_pitch;
1084 return true;
1087 static inline bool copy_rect( filter_sys_t* p_sys,
1088 uint16_t i_x, uint16_t i_y,
1089 uint16_t i_w, uint16_t i_h,
1090 uint16_t i_sx, uint16_t i_sy )
1092 plane_t *p_Y = p_sys->p_pic->p+Y_PLANE;
1093 plane_t *p_U = p_sys->p_pic->p+U_PLANE;
1094 plane_t *p_V = p_sys->p_pic->p+V_PLANE;
1095 plane_t *p_A = p_sys->p_pic->p+A_PLANE;
1097 int i_pitch = p_Y->i_pitch;
1098 int i_lines = p_Y->i_lines;
1100 fprintf( stderr, "copy_rect: (%d,%d)+(%d,%d) -> (%d,%d)\n", i_x, i_y, i_w, i_h, i_sx, i_sy );
1102 if( i_x + i_w > i_pitch || i_sx + i_w > i_pitch )
1103 return false;
1104 if( i_y + i_h > i_lines || i_sy + i_h > i_lines)
1105 return false;
1107 if( i_w <= 0 || i_h <= 0 )
1108 return true;
1110 uint8_t *pb_buffer = calloc( i_w * i_h, 4 );
1111 if( !pb_buffer )
1112 return false;
1114 for( int i_line = 0; i_line < i_h; i_line++ )
1116 for( int i_column = 0; i_column < i_w; i_column++ )
1118 const int i_src_offset = ( i_sy + i_line ) * i_pitch + i_sx + i_column;
1119 const int i_tmp_offset = ( 0 + i_line ) * i_w + 0 + i_column;
1121 pb_buffer[4*i_tmp_offset + 0] = p_Y->p_pixels[i_src_offset];
1122 pb_buffer[4*i_tmp_offset + 1] = p_U->p_pixels[i_src_offset];
1123 pb_buffer[4*i_tmp_offset + 2] = p_V->p_pixels[i_src_offset];
1124 pb_buffer[4*i_tmp_offset + 3] = p_A->p_pixels[i_src_offset];
1128 for( int i_line = 0; i_line < i_h; i_line++ )
1130 for( int i_column = 0; i_column < i_w; i_column++ )
1132 const int i_tmp_offset = ( 0 + i_line ) * i_w + 0 + i_column;
1133 const int i_dst_offset = ( i_y + i_line ) * i_pitch + i_x + i_column;
1135 p_Y->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 0];
1136 p_U->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 1];
1137 p_V->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 2];
1138 p_A->p_pixels[i_dst_offset] = pb_buffer[4*i_tmp_offset + 3];
1141 free( pb_buffer );
1142 return true;
1146 static inline bool raw_line( filter_sys_t* p_sys,
1147 uint16_t i_x, uint16_t i_y,
1148 uint16_t i_w )
1150 plane_t *p_outY = p_sys->p_pic->p+Y_PLANE;
1151 plane_t *p_outU = p_sys->p_pic->p+U_PLANE;
1152 plane_t *p_outV = p_sys->p_pic->p+V_PLANE;
1153 plane_t *p_outA = p_sys->p_pic->p+A_PLANE;
1154 int i_pitch = p_outY->i_pitch;
1155 int i_lines = p_outY->i_lines;
1156 if ( i_x + i_w > i_pitch)
1157 return false;
1158 if ( i_y > i_lines)
1159 return false;
1161 int i_line_offset = i_y * i_pitch + i_x;
1163 for( int i_column = 0; i_column < i_w; i_column++ )
1165 int i_offset = i_line_offset + i_column;
1166 uint8_t i_color = p_sys->read_buffer[i_column];
1167 p_outY->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][0];
1168 p_outU->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][1];
1169 p_outV->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][2];
1170 p_outA->p_pixels[ i_offset ] = p_sys->ar_color_table_yuv[i_color][3];
1173 return true;
1177 /*****************************************************************************
1178 * MouseEvent: callback for mouse events
1179 *****************************************************************************/
1180 static int MouseEvent( filter_t *p_filter,
1181 const vlc_mouse_t *p_old,
1182 const vlc_mouse_t *p_new,
1183 const video_format_t *p_fmt )
1185 filter_sys_t *p_sys = p_filter->p_sys;
1186 VLC_UNUSED(p_old);
1188 int i_v = p_new->i_pressed;
1189 int i_x = p_new->i_x;
1190 int i_y = p_new->i_y;
1192 vlc_mutex_lock( &p_sys->lock );
1194 const int v_h = p_fmt->i_visible_height;
1195 const int v_w = p_sys->i_vnc_width * v_h / p_sys->i_vnc_height;
1196 const int v_x = (p_fmt->i_visible_width-v_w)/2;
1198 i_x -= v_x;
1200 if( i_y < 0 || i_x < 0 || i_y >= v_h || i_x >= v_w )
1202 vlc_mutex_unlock( &p_sys->lock );
1203 msg_Dbg( p_filter, "invalid mouse event? x=%d y=%d btn=%x", i_x, i_y, i_v );
1204 return VLC_SUCCESS;
1207 if( p_sys->i_socket == -1 )
1209 vlc_mutex_unlock( &p_sys->lock );
1210 return VLC_SUCCESS;
1213 #ifdef VNC_DEBUG
1214 msg_Dbg( p_filter, "mouse event x=%d y=%d btn=%x", i_x, i_y, i_v );
1215 #endif
1217 /* */
1218 i_x = i_x * p_sys->i_vnc_width / v_w;
1219 i_y = i_y * p_sys->i_vnc_height / v_h;
1221 /* buttonMask bits 0-7 are buttons 1-8, 0=up, 1=down */
1222 rfbPointerEventMsg ev;
1223 ev.type = rfbPointerEvent;
1224 ev.buttonMask = i_v;
1225 ev.x = htons(i_x);
1226 ev.y = htons(i_y);
1228 write_exact( p_filter, p_sys->i_socket, &ev, sz_rfbPointerEventMsg);
1229 vlc_mutex_unlock( &p_sys->lock );
1231 return VLC_EGENERIC;
1234 /*****************************************************************************
1235 * KeyEvent: callback for keyboard events
1236 *****************************************************************************/
1237 static int KeyEvent( vlc_object_t *p_this, char const *psz_var,
1238 vlc_value_t oldval, vlc_value_t newval, void *p_data )
1240 VLC_UNUSED(psz_var); VLC_UNUSED(oldval);
1242 filter_t *p_filter = (filter_t *)p_data;
1243 filter_sys_t *p_sys = p_filter->p_sys;
1245 msg_Dbg( p_this, "key pressed (%"PRId64") ", newval.i_int );
1247 if ( !newval.i_int )
1249 msg_Err( p_this, "Received invalid key event 0" );
1250 return VLC_EGENERIC;
1253 vlc_mutex_lock( &p_sys->lock );
1254 if( p_sys->i_socket == -1 )
1256 vlc_mutex_unlock( &p_sys->lock );
1257 return VLC_SUCCESS;
1260 uint32_t i_key32 = newval.i_int;
1261 i_key32 = htonl(i_key32);
1262 rfbKeyEventMsg ev;
1264 ev.type = rfbKeyEvent;
1265 ev.down = 1;
1266 ev.pad = 0;
1268 /* first key-down for modifier-keys */
1269 if (newval.i_int & KEY_MODIFIER_CTRL)
1271 ev.key = 0xffe3;
1272 write_exact( p_filter, p_sys->i_socket, &ev, sz_rfbKeyEventMsg );
1274 if (newval.i_int & KEY_MODIFIER_SHIFT)
1276 ev.key = 0xffe1;
1277 write_exact( p_filter, p_sys->i_socket, &ev, sz_rfbKeyEventMsg );
1279 if (newval.i_int & KEY_MODIFIER_ALT)
1281 ev.key = 0xffe9;
1282 write_exact( p_filter, p_sys->i_socket, &ev, sz_rfbKeyEventMsg );
1285 /* then key-down for the pressed key */
1286 ev.key = i_key32;
1287 write_exact( p_filter, p_sys->i_socket, &ev, sz_rfbKeyEventMsg );
1289 ev.down = 0;
1291 /* then key-up for the pressed key */
1292 write_exact( p_filter, p_sys->i_socket, &ev, sz_rfbKeyEventMsg );
1294 /* last key-down for modifier-keys */
1295 if (newval.i_int & KEY_MODIFIER_CTRL)
1297 ev.key = 0xffe3;
1298 write_exact( p_filter, p_sys->i_socket, &ev, sz_rfbKeyEventMsg );
1300 if (newval.i_int & KEY_MODIFIER_SHIFT)
1302 ev.key = 0xffe1;
1303 write_exact( p_filter, p_sys->i_socket, &ev, sz_rfbKeyEventMsg );
1305 if (newval.i_int & KEY_MODIFIER_ALT)
1307 ev.key = 0xffe9;
1308 write_exact( p_filter, p_sys->i_socket, &ev, sz_rfbKeyEventMsg );
1310 vlc_mutex_unlock( &p_sys->lock );
1312 return VLC_SUCCESS;
1315 static void vnc_encrypt_bytes( unsigned char *bytes, char *passwd )
1317 unsigned char key[8];
1319 for( unsigned i = 0; i < 8; i++ )
1320 key[i] = i < strlen( passwd ) ? passwd[i] : '\0';
1322 gcry_cipher_hd_t ctx;
1323 gcry_cipher_open( &ctx, GCRY_CIPHER_DES, GCRY_CIPHER_MODE_ECB,0);
1325 /* reverse bits of the key */
1326 for( unsigned i = 0 ; i < 8 ; i++ )
1327 key[i] =
1328 (key[i] >> 7) +
1329 (((key[i] >> 6) & 0x01 ) << 1 ) +
1330 (((key[i] >> 5) & 0x01 ) << 2 ) +
1331 (((key[i] >> 4) & 0x01 ) << 3 ) +
1332 (((key[i] >> 3) & 0x01 ) << 4 ) +
1333 (((key[i] >> 2) & 0x01 ) << 5 ) +
1334 (((key[i] >> 1) & 0x01 ) << 6 ) +
1335 ((key[i] & 0x01) << 7 );
1337 gcry_cipher_setkey( ctx, key, 8 );
1338 gcry_cipher_encrypt( ctx, bytes, CHALLENGESIZE, bytes, CHALLENGESIZE );
1339 gcry_cipher_close( ctx );