4 * Routines to implement Compact Rise-and-Run-length Encoding (CoRRE). This
5 * code is based on krw's original javatel rfbserver.
9 * OSXvnc Copyright (C) 2001 Dan McGuirk <mcguirk@incompleteness.net>.
10 * Original Xvnc code Copyright (C) 1999 AT&T Laboratories Cambridge.
11 * All Rights Reserved.
13 * This is free software; you can redistribute it and/or modify
14 * it under the terms of the GNU General Public License as published by
15 * the Free Software Foundation; either version 2 of the License, or
16 * (at your option) any later version.
18 * This software is distributed in the hope that it will be useful,
19 * but WITHOUT ANY WARRANTY; without even the implied warranty of
20 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
21 * GNU General Public License for more details.
23 * You should have received a copy of the GNU General Public License
24 * along with this software; if not, write to the Free Software
25 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
34 * rreBeforeBuf contains pixel data in the client's format.
35 * rreAfterBuf contains the RRE encoded version. If the RRE encoded version is
36 * larger than the raw data or if it exceeds rreAfterBufSize then
37 * raw encoding is used instead.
40 static int rreBeforeBufSize
= 0;
41 static char *rreBeforeBuf
= NULL
;
43 static int rreAfterBufSize
= 0;
44 static char *rreAfterBuf
= NULL
;
45 static int rreAfterBufLen
;
47 static int subrectEncode8(CARD8
*data
, int w
, int h
);
48 static int subrectEncode16(CARD16
*data
, int w
, int h
);
49 static int subrectEncode32(CARD32
*data
, int w
, int h
);
50 static CARD32
getBgColour(char *data
, int size
, int bpp
);
51 static Bool
rfbSendSmallRectEncodingCoRRE(rfbClientPtr cl
, int x
, int y
,
56 * rfbSendRectEncodingCoRRE - send an arbitrary size rectangle using CoRRE
61 rfbSendRectEncodingCoRRE(cl
, x
, y
, w
, h
)
65 if (h
> cl
->correMaxHeight
) {
66 rfbSendRectEncodingCoRRE(cl
, x
, y
, w
, cl
->correMaxHeight
);
67 rfbSendRectEncodingCoRRE(cl
, x
, y
+ cl
->correMaxHeight
, w
,
68 h
- cl
->correMaxHeight
);
72 if (w
> cl
->correMaxWidth
) {
73 rfbSendRectEncodingCoRRE(cl
, x
, y
, cl
->correMaxWidth
, h
);
74 rfbSendRectEncodingCoRRE(cl
, x
+ cl
->correMaxWidth
, y
,
75 w
- cl
->correMaxWidth
, h
);
79 rfbSendSmallRectEncodingCoRRE(cl
, x
, y
, w
, h
);
86 * rfbSendSmallRectEncodingCoRRE - send a small (guaranteed < 256x256)
87 * rectangle using CoRRE encoding.
91 rfbSendSmallRectEncodingCoRRE(cl
, x
, y
, w
, h
)
95 rfbFramebufferUpdateRectHeader rect
;
99 char *fbptr
= (cl
->screen
->frameBuffer
+ (cl
->screen
->paddedWidthInBytes
* y
)
100 + (x
* (cl
->screen
->bitsPerPixel
/ 8)));
102 int maxRawSize
= (cl
->screen
->width
* cl
->screen
->height
103 * (cl
->format
.bitsPerPixel
/ 8));
105 if (rreBeforeBufSize
< maxRawSize
) {
106 rreBeforeBufSize
= maxRawSize
;
107 if (rreBeforeBuf
== NULL
)
108 rreBeforeBuf
= (char *)malloc(rreBeforeBufSize
);
110 rreBeforeBuf
= (char *)realloc(rreBeforeBuf
, rreBeforeBufSize
);
113 if (rreAfterBufSize
< maxRawSize
) {
114 rreAfterBufSize
= maxRawSize
;
115 if (rreAfterBuf
== NULL
)
116 rreAfterBuf
= (char *)malloc(rreAfterBufSize
);
118 rreAfterBuf
= (char *)realloc(rreAfterBuf
, rreAfterBufSize
);
121 (*cl
->translateFn
)(cl
->translateLookupTable
,&(cl
->screen
->rfbServerFormat
),
122 &cl
->format
, fbptr
, rreBeforeBuf
,
123 cl
->screen
->paddedWidthInBytes
, w
, h
);
125 switch (cl
->format
.bitsPerPixel
) {
127 nSubrects
= subrectEncode8((CARD8
*)rreBeforeBuf
, w
, h
);
130 nSubrects
= subrectEncode16((CARD16
*)rreBeforeBuf
, w
, h
);
133 nSubrects
= subrectEncode32((CARD32
*)rreBeforeBuf
, w
, h
);
136 rfbLog("getBgColour: bpp %d?\n",cl
->format
.bitsPerPixel
);
142 /* RRE encoding was too large, use raw */
144 return rfbSendRectEncodingRaw(cl
, x
, y
, w
, h
);
147 cl
->rfbRectanglesSent
[rfbEncodingCoRRE
]++;
148 cl
->rfbBytesSent
[rfbEncodingCoRRE
] += (sz_rfbFramebufferUpdateRectHeader
149 + sz_rfbRREHeader
+ rreAfterBufLen
);
151 if (cl
->ublen
+ sz_rfbFramebufferUpdateRectHeader
+ sz_rfbRREHeader
154 if (!rfbSendUpdateBuf(cl
))
158 rect
.r
.x
= Swap16IfLE(x
);
159 rect
.r
.y
= Swap16IfLE(y
);
160 rect
.r
.w
= Swap16IfLE(w
);
161 rect
.r
.h
= Swap16IfLE(h
);
162 rect
.encoding
= Swap32IfLE(rfbEncodingCoRRE
);
164 memcpy(&cl
->updateBuf
[cl
->ublen
], (char *)&rect
,
165 sz_rfbFramebufferUpdateRectHeader
);
166 cl
->ublen
+= sz_rfbFramebufferUpdateRectHeader
;
168 hdr
.nSubrects
= Swap32IfLE(nSubrects
);
170 memcpy(&cl
->updateBuf
[cl
->ublen
], (char *)&hdr
, sz_rfbRREHeader
);
171 cl
->ublen
+= sz_rfbRREHeader
;
173 for (i
= 0; i
< rreAfterBufLen
;) {
175 int bytesToCopy
= UPDATE_BUF_SIZE
- cl
->ublen
;
177 if (i
+ bytesToCopy
> rreAfterBufLen
) {
178 bytesToCopy
= rreAfterBufLen
- i
;
181 memcpy(&cl
->updateBuf
[cl
->ublen
], &rreAfterBuf
[i
], bytesToCopy
);
183 cl
->ublen
+= bytesToCopy
;
186 if (cl
->ublen
== UPDATE_BUF_SIZE
) {
187 if (!rfbSendUpdateBuf(cl
))
198 * subrectEncode() encodes the given multicoloured rectangle as a background
199 * colour overwritten by single-coloured rectangles. It returns the number
200 * of subrectangles in the encoded buffer, or -1 if subrect encoding won't
201 * fit in the buffer. It puts the encoded rectangles in rreAfterBuf. The
202 * single-colour rectangle partition is not optimal, but does find the biggest
203 * horizontal or vertical rectangle top-left anchored to each consecutive
204 * coordinate position.
206 * The coding scheme is simply [<bgcolour><subrect><subrect>...] where each
207 * <subrect> is [<colour><x><y><w><h>].
210 #define DEFINE_SUBRECT_ENCODE(bpp) \
212 subrectEncode##bpp(data,w,h) \
218 rfbCoRRERectangle subrect; \
221 int hx=0,hy,vx=0,vy; \
226 int thex,they,thew,theh; \
229 CARD##bpp bg = (CARD##bpp)getBgColour((char*)data,w*h,bpp); \
231 *((CARD##bpp*)rreAfterBuf) = bg; \
233 rreAfterBufLen = (bpp/8); \
235 for (y=0; y<h; y++) { \
237 for (x=0; x<w; x++) { \
238 if (line[x] != bg) { \
242 for (j=y; j<h; j++) { \
244 if (seg[x] != cl) {break;} \
246 while ((seg[i] == cl) && (i < w)) i += 1; \
248 if (j == y) vx = hx = i; \
249 if (i < vx) vx = i; \
250 if ((hyflag > 0) && (i >= hx)) {hy += 1;} else {hyflag = 0;} \
254 /* We now have two possible subrects: (x,y,hx,hy) and (x,y,vx,vy) \
255 * We'll choose the bigger of the two. \
265 if ((hw*hh) > (vw*vh)) { \
278 newLen = rreAfterBufLen + (bpp/8) + sz_rfbCoRRERectangle; \
279 if ((newLen > (w * h * (bpp/8))) || (newLen > rreAfterBufSize)) \
283 *((CARD##bpp*)(rreAfterBuf + rreAfterBufLen)) = cl; \
284 rreAfterBufLen += (bpp/8); \
285 memcpy(&rreAfterBuf[rreAfterBufLen],&subrect,sz_rfbCoRRERectangle); \
286 rreAfterBufLen += sz_rfbCoRRERectangle; \
289 * Now mark the subrect as done. \
291 for (j=they; j < (they+theh); j++) { \
292 for (i=thex; i < (thex+thew); i++) { \
303 DEFINE_SUBRECT_ENCODE(8)
304 DEFINE_SUBRECT_ENCODE(16)
305 DEFINE_SUBRECT_ENCODE(32)
309 * getBgColour() gets the most prevalent colour in a byte array.
312 getBgColour(data
,size
,bpp
)
320 static int counts
[NUMCLRS
];
328 return ((CARD16
*)data
)[0];
329 } else if (bpp
== 32) {
330 return ((CARD32
*)data
)[0];
332 rfbLog("getBgColour: bpp %d?\n",bpp
);
337 for (i
=0; i
<NUMCLRS
; i
++) {
341 for (j
=0; j
<size
; j
++) {
342 k
= (int)(((CARD8
*)data
)[j
]);
344 rfbLog("getBgColour: unusual colour = %d\n", k
);
348 if (counts
[k
] > maxcount
) {
349 maxcount
= counts
[k
];
350 maxclr
= ((CARD8
*)data
)[j
];