uLab
/
xrdp-proprietary
mirror of https://mirror.git.trinitydesktop.org/gitea/uLab/xrdp-proprietary
0
0
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #549 from jsorg71/rlelimit

Browse Source 
fixes for rle bitmap compress limits and raw bitmaps
master
jsorg71 8 years ago committed by GitHub
parent d706f9dba4 492116d535
commit 9584e22ba6
3 changed files with 153 additions and 92 deletions
Show Stats Download Patch File Download Diff File

122
libxrdp/xrdp_bitmap32_compress.c
Unescape View File

@ -107,7 +107,7 @@ fsplit3(char *in_data, int start_line, int width, int e,
} }
start_line--; start_line--;
cy++; cy++;
if (out_index > 64 * 64) if (out_index + width + e > 64 * 64)
{ {
break; break;
} }
@ -195,7 +195,7 @@ fsplit4(char *in_data, int start_line, int width, int e,
} }
start_line--; start_line--;
cy++; cy++;
if (out_index > 64 * 64) if (out_index + width + e > 64 * 64)
{ {
break; break;
} }
@ -422,6 +422,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
char *sr_data; char *sr_data;
char *sg_data; char *sg_data;
char *sb_data; char *sb_data;
char *hold_p;
int a_bytes; int a_bytes;
int r_bytes; int r_bytes;
int g_bytes; int g_bytes;
@ -449,6 +450,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
r_data = a_data + max_bytes; r_data = a_data + max_bytes;
g_data = r_data + max_bytes; g_data = r_data + max_bytes;
b_data = g_data + max_bytes; b_data = g_data + max_bytes;
hold_p = s->p;
if (header & FLAGS_NOALPHA) if (header & FLAGS_NOALPHA)
{ {
@ -459,35 +461,44 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
fdelta(sr_data, r_data, cx, cy); fdelta(sr_data, r_data, cx, cy);
fdelta(sg_data, g_data, cx, cy); fdelta(sg_data, g_data, cx, cy);
fdelta(sb_data, b_data, cx, cy); fdelta(sb_data, b_data, cx, cy);
out_uint8(s, header); while (cy > 0)
r_bytes = fpack(r_data, cx, cy, s);
g_bytes = fpack(g_data, cx, cy, s);
b_bytes = fpack(b_data, cx, cy, s);
total_bytes = r_bytes + g_bytes + b_bytes;
if (1 + total_bytes > byte_limit)
{ {
/* failed */ s->p = hold_p;
LLOGLN(0, ("xrdp_bitmap32_compress: too big, rgb " out_uint8(s, header);
"bytes %d %d %d total_bytes %d cx %d cy %d " r_bytes = fpack(r_data, cx, cy, s);
"byte_limit %d", r_bytes, g_bytes, b_bytes, g_bytes = fpack(g_data, cx, cy, s);
total_bytes, cx, cy, byte_limit)); b_bytes = fpack(b_data, cx, cy, s);
return 0; max_bytes = cx * cy * 3;
} total_bytes = r_bytes + g_bytes + b_bytes;
max_bytes = cx * cy * 3; if (total_bytes > max_bytes)
if (total_bytes > max_bytes) {
{ if (2 + max_bytes <= byte_limit)
/* raw is better */ {
LLOGLN(10, ("xrdp_bitmap32_compress: too big, rgb " s->p = hold_p;
"bytes %d %d %d total_bytes %d cx %d cy %d " foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
"max_bytes %d", r_bytes, g_bytes, b_bytes, break;
total_bytes, cx, cy, max_bytes)); }
init_stream(s, 0); }
foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); if (1 + total_bytes <= byte_limit)
{
break;
}
cy--;
} }
} }
else else
{ {
foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); while (cy > 0)
{
max_bytes = cx * cy * 3;
if (2 + max_bytes <= byte_limit)
{
s->p = hold_p;
foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
break;
}
cy--;
}
} }
} }
else else
@ -500,36 +511,45 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
fdelta(sr_data, r_data, cx, cy); fdelta(sr_data, r_data, cx, cy);
fdelta(sg_data, g_data, cx, cy); fdelta(sg_data, g_data, cx, cy);
fdelta(sb_data, b_data, cx, cy); fdelta(sb_data, b_data, cx, cy);
out_uint8(s, header); while (cy > 0)
a_bytes = fpack(a_data, cx, cy, s);
r_bytes = fpack(r_data, cx, cy, s);
g_bytes = fpack(g_data, cx, cy, s);
b_bytes = fpack(b_data, cx, cy, s);
max_bytes = cx * cy * 4;
total_bytes = a_bytes + r_bytes + g_bytes + b_bytes;
if (1 + total_bytes > byte_limit)
{ {
/* failed */ s->p = hold_p;
LLOGLN(0, ("xrdp_bitmap32_compress: too big, argb " out_uint8(s, header);
"bytes %d %d %d %d total_bytes %d cx %d cy %d " a_bytes = fpack(a_data, cx, cy, s);
"byte_limit %d", a_bytes, r_bytes, g_bytes, b_bytes, r_bytes = fpack(r_data, cx, cy, s);
total_bytes, cx, cy, byte_limit)); g_bytes = fpack(g_data, cx, cy, s);
return 0; b_bytes = fpack(b_data, cx, cy, s);
} max_bytes = cx * cy * 4;
if (total_bytes > max_bytes) total_bytes = a_bytes + r_bytes + g_bytes + b_bytes;
{ if (total_bytes > max_bytes)
/* raw is better */ {
LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb " if (2 + max_bytes <= byte_limit)
"bytes %d %d %d %d total_bytes %d cx %d cy %d " {
"max_bytes %d", a_bytes, r_bytes, g_bytes, b_bytes, s->p = hold_p;
total_bytes, cx, cy, max_bytes)); foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
init_stream(s, 0); break;
foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); }
}
if (1 + total_bytes <= byte_limit)
{
break;
}
cy--;
} }
} }
else else
{ {
foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); while (cy > 0)
{
max_bytes = cx * cy * 4;
if (2 + max_bytes <= byte_limit)
{
s->p = hold_p;
foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
break;
}
cy--;
}
} }
} }
return cy; return cy;

8
libxrdp/xrdp_bitmap_compress.c
Unescape View File

@ -22,6 +22,8 @@
#include "libxrdp.h" #include "libxrdp.h"
#define BC_MAX_BYTES (16 * 1024)
/*****************************************************************************/ /*****************************************************************************/
#define IN_PIXEL8(in_ptr, in_x, in_y, in_w, in_last_pixel, in_pixel); \ #define IN_PIXEL8(in_ptr, in_x, in_y, in_w, in_last_pixel, in_pixel); \
do { \ do { \
@ -695,7 +697,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
out_count = end; out_count = end;
line = in_data + width * start_line; line = in_data + width * start_line;
while (start_line >= 0 && out_count < 32768) while (start_line >= 0 && out_count <= BC_MAX_BYTES)
{ {
i = (s->p - s->data) + count; i = (s->p - s->data) + count;
@ -987,7 +989,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
out_count = end * 2; out_count = end * 2;
line = in_data + width * start_line * 2; line = in_data + width * start_line * 2;
while (start_line >= 0 && out_count < 32768) while (start_line >= 0 && out_count <= BC_MAX_BYTES)
{ {
i = (s->p - s->data) + count * 2; i = (s->p - s->data) + count * 2;
@ -1279,7 +1281,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
out_count = end * 3; out_count = end * 3;
line = in_data + width * start_line * 4; line = in_data + width * start_line * 4;
while (start_line >= 0 && out_count < 32768) while (start_line >= 0 && out_count <= BC_MAX_BYTES)
{ {
i = (s->p - s->data) + count * 3; i = (s->p - s->data) + count * 3;

115
libxrdp/xrdp_orders.c
Unescape View File

@ -2235,6 +2235,11 @@ xrdp_orders_send_raw_bitmap(struct xrdp_orders *self,
Bpp = (bpp + 7) / 8; Bpp = (bpp + 7) / 8;
bufsize = (width + e) * height * Bpp; bufsize = (width + e) * height * Bpp;
while (bufsize + 16 > MAX_ORDERS_SIZE)
{
height--;
bufsize = (width + e) * height * Bpp;
}
if (xrdp_orders_check(self, bufsize + 16) != 0) if (xrdp_orders_check(self, bufsize + 16) != 0)
{ {
return 1; return 1;
@ -2254,33 +2259,58 @@ xrdp_orders_send_raw_bitmap(struct xrdp_orders *self,
out_uint16_le(self->out_s, bufsize); out_uint16_le(self->out_s, bufsize);
out_uint16_le(self->out_s, cache_idx); out_uint16_le(self->out_s, cache_idx);
for (i = height - 1; i >= 0; i--) if (Bpp == 4)
{ {
for (j = 0; j < width; j++) for (i = height - 1; i >= 0; i--)
{ {
if (Bpp == 3) for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL32(data, j, i, width); pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16); out_uint8(self->out_s, pixel >> 16);
out_uint8(self->out_s, pixel >> 24);
} }
else if (Bpp == 2) out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 3)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL16(data, j, i, width); pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16);
} }
else if (Bpp == 1) out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 2)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL8(data, j, i, width); pixel = GETPIXEL16(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8);
} }
out_uint8s(self->out_s, Bpp * e);
} }
}
for (j = 0; j < e; j++) else if (Bpp == 1)
{
for (i = height - 1; i >= 0; i--)
{ {
out_uint8s(self->out_s, Bpp); for (j = 0; j < width; j++)
{
pixel = GETPIXEL8(data, j, i, width);
out_uint8(self->out_s, pixel);
}
out_uint8s(self->out_s, Bpp * e);
} }
} }
@ -2334,20 +2364,19 @@ xrdp_orders_send_bitmap(struct xrdp_orders *self,
if (bpp > 24) if (bpp > 24)
{ {
lines_sending = xrdp_bitmap32_compress(data, width, height, s, lines_sending = xrdp_bitmap32_compress(data, width, height, s,
bpp, 16384, bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e, 0x10); i - 1, temp_s, e, 0x10);
} }
else else
{ {
lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384, lines_sending = xrdp_bitmap_compress(data, width, height, s,
bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e); i - 1, temp_s, e);
} }
if (lines_sending != height) if (lines_sending != height)
{ {
g_writeln("error in xrdp_orders_send_bitmap, lines_sending(%d) != \ height = lines_sending;
height(%d)", lines_sending, height);
return 1;
} }
bufsize = (int)(s->p - p); bufsize = (int)(s->p - p);
@ -2458,6 +2487,7 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
int pixel = 0; int pixel = 0;
int e = 0; int e = 0;
g_writeln("xrdp_orders_send_raw_bitmap2:");
if (width > 64) if (width > 64)
{ {
g_writeln("error, width > 64"); g_writeln("error, width > 64");
@ -2479,6 +2509,11 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
Bpp = (bpp + 7) / 8; Bpp = (bpp + 7) / 8;
bufsize = (width + e) * height * Bpp; bufsize = (width + e) * height * Bpp;
while (bufsize + 14 > MAX_ORDERS_SIZE)
{
height--;
bufsize = (width + e) * height * Bpp;
}
if (xrdp_orders_check(self, bufsize + 14) != 0) if (xrdp_orders_check(self, bufsize + 14) != 0)
{ {
return 1; return 1;
@ -2499,7 +2534,7 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
i = cache_idx & 0xff; i = cache_idx & 0xff;
out_uint8(self->out_s, i); out_uint8(self->out_s, i);
if (1 && Bpp == 3) if (Bpp == 4)
{ {
for (i = height - 1; i >= 0; i--) for (i = height - 1; i >= 0; i--)
{ {
@ -2509,44 +2544,49 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16); out_uint8(self->out_s, pixel >> 16);
out_uint8(self->out_s, pixel >> 24);
} }
for (j = 0; j < e; j++) out_uint8s(self->out_s, Bpp * e);
{
out_uint8s(self->out_s, Bpp);
}
} }
} }
else else if (Bpp == 3)
{
for (i = height - 1; i >= 0; i--)
{ {
for (j = 0; j < width; j++) for (i = height - 1; i >= 0; i--)
{ {
if (Bpp == 3) for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL32(data, j, i, width); pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16); out_uint8(self->out_s, pixel >> 16);
} }
else if (Bpp == 2) out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 2)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL16(data, j, i, width); pixel = GETPIXEL16(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 8);
} }
else if (Bpp == 1) out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 1)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL8(data, j, i, width); pixel = GETPIXEL8(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
} }
out_uint8s(self->out_s, Bpp * e);
} }
for (j = 0; j < e; j++)
{
out_uint8s(self->out_s, Bpp);
}
}
} }
return 0; return 0;
@ -2599,20 +2639,19 @@ xrdp_orders_send_bitmap2(struct xrdp_orders *self,
if (bpp > 24) if (bpp > 24)
{ {
lines_sending = xrdp_bitmap32_compress(data, width, height, s, lines_sending = xrdp_bitmap32_compress(data, width, height, s,
bpp, 16384, bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e, 0x10); i - 1, temp_s, e, 0x10);
} }
else else
{ {
lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384, lines_sending = xrdp_bitmap_compress(data, width, height, s,
bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e); i - 1, temp_s, e);
} }
if (lines_sending != height) if (lines_sending != height)
{ {
g_writeln("error in xrdp_orders_send_bitmap2, lines_sending(%d) != \ height = lines_sending;
height(%d)", lines_sending, height);
return 1;
} }
bufsize = (int)(s->p - p); bufsize = (int)(s->p - p);

Write Preview
Loading…
Cancel
Save