From aa35a327528c507505de33496dd214c40675fd6e Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Sun, 15 Jun 2014 16:17:40 -0700 Subject: [PATCH] libxrdp: 32 bpp compression changes / fixes --- libxrdp/xrdp_bitmap32_compress.c | 151 +++++++++++++++++++------------ libxrdp/xrdp_orders.c | 12 ++- 2 files changed, 105 insertions(+), 58 deletions(-) diff --git a/libxrdp/xrdp_bitmap32_compress.c b/libxrdp/xrdp_bitmap32_compress.c index c3afff90..1bcf5db9 100644 --- a/libxrdp/xrdp_bitmap32_compress.c +++ b/libxrdp/xrdp_bitmap32_compress.c @@ -35,6 +35,58 @@ http://msdn.microsoft.com/en-us/library/cc241877.aspx #define LHEXDUMP(_level, _args) \ do { if (_level < LLOG_LEVEL) { g_hexdump _args ; } } while (0) +/*****************************************************************************/ +static int APP_CC +fsplit(char *in_data, int start_line, int width, int e, + char *alpha_data, char *red_data, char *green_data, char *blue_data) +{ + int index; + int pixel; + int cy; + int alpha_bytes; + int red_bytes; + int green_bytes; + int blue_bytes; + int *ptr32; + + cy = 0; + alpha_bytes = 0; + red_bytes = 0; + green_bytes = 0; + blue_bytes = 0; + while (start_line >= 0) + { + ptr32 = (int *) (in_data + start_line * width * 4); + for (index = 0; index < width; index++) + { + pixel = *ptr32; + ptr32++; + alpha_data[alpha_bytes] = pixel >> 24; + alpha_bytes++; + red_data[red_bytes] = pixel >> 16; + red_bytes++; + green_data[green_bytes] = pixel >> 8; + green_bytes++; + blue_data[blue_bytes] = pixel >> 0; + blue_bytes++; + } + for (index = 0; index < e; index++) + { + alpha_data[alpha_bytes] = 0; + alpha_bytes++; + red_data[red_bytes] = 0; + red_bytes++; + green_data[green_bytes] = 0; + green_bytes++; + blue_data[blue_bytes] = 0; + blue_bytes++; + } + start_line--; + cy++; + } + return cy; +} + /*****************************************************************************/ static int APP_CC fdelta(char *plane, int cx, int cy) @@ -135,11 +187,13 @@ fpack(char *plane, int cx, int cy, struct stream *s) char *ptr8; char *colptr; char *lend; + char *holdp; int jndex; int collen; int replen; LLOGLN(10, ("fpack:")); + holdp = s->p; for (jndex = 0; jndex < cy; jndex++) { LLOGLN(10, ("line start line %d cx %d cy %d", jndex, cx, cy)); @@ -190,7 +244,7 @@ fpack(char *plane, int cx, int cy, struct stream *s) /* end of line */ fout(collen, replen, colptr, s); } - return 0; + return (int) (s->p - holdp); } /*****************************************************************************/ @@ -201,8 +255,6 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, int start_line, struct stream *temp_s, int e) { - int pixel; - int *ptr32; char *alpha_data; char *red_data; char *green_data; @@ -211,10 +263,10 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, int red_bytes; int green_bytes; int blue_bytes; - int index; int cx; int cy; int header; + int max_bytes; LLOGLN(10, ("xrdp_bitmap32_compress:")); @@ -223,80 +275,67 @@ xrdp_bitmap32_compress(char *in_data, int width, int height, header = FLAGS_RLE; cx = width + e; - cy = 0; alpha_data = temp_s->data; red_data = alpha_data + cx * height; green_data = red_data + cx * height; blue_data = green_data + cx * height; - alpha_bytes = 0; - red_bytes = 0; - green_bytes = 0; - blue_bytes = 0; /* split planes */ - while (start_line >= 0) - { - ptr32 = (int *) (in_data + start_line * width * 4); - for (index = 0; index < width; index++) - { - pixel = *ptr32; - ptr32++; - alpha_data[alpha_bytes] = pixel >> 24; - alpha_bytes++; - red_data[red_bytes] = pixel >> 16; - red_bytes++; - green_data[green_bytes] = pixel >> 8; - green_bytes++; - blue_data[blue_bytes] = pixel >> 0; - blue_bytes++; - } - for (index = 0; index < e; index++) - { - alpha_data[alpha_bytes] = 0; - alpha_bytes++; - red_data[red_bytes] = 0; - red_bytes++; - green_data[green_bytes] = 0; - green_bytes++; - blue_data[blue_bytes] = 0; - blue_bytes++; - } - start_line--; - cy++; - } + cy = fsplit(in_data, start_line, width, e, + alpha_data, red_data, green_data, blue_data); if (header & FLAGS_RLE) { out_uint8(s, header); - - /* delta, other steps */ - if ((header & FLAGS_NOALPHA) == 0) + if (header & FLAGS_NOALPHA) + { + fdelta(red_data, cx, cy); + fdelta(green_data, cx, cy); + fdelta(blue_data, cx, cy); + red_bytes = fpack(red_data, cx, cy, s); + green_bytes = fpack(green_data, cx, cy, s); + blue_bytes = fpack(blue_data, cx, cy, s); + max_bytes = cx * cy * 3; + } + else { fdelta(alpha_data, cx, cy); + fdelta(red_data, cx, cy); + fdelta(green_data, cx, cy); + fdelta(blue_data, cx, cy); + alpha_bytes = fpack(alpha_data, cx, cy, s); + red_bytes = fpack(red_data, cx, cy, s); + green_bytes = fpack(green_data, cx, cy, s); + blue_bytes = fpack(blue_data, cx, cy, s); + max_bytes = cx * cy * 4; } - fdelta(red_data, cx, cy); - fdelta(green_data, cx, cy); - fdelta(blue_data, cx, cy); - - /* pack */ - if ((header & FLAGS_NOALPHA) == 0) + if (alpha_bytes + red_bytes + green_bytes + blue_bytes > max_bytes) { - fpack(alpha_data, cx, cy, s); + LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb " + "bytes %d %d %d %d cx %d cy %d", alpha_bytes, red_bytes, + green_bytes, blue_bytes, cx, cy)); } - fpack(red_data, cx, cy, s); - fpack(green_data, cx, cy, s); - fpack(blue_data, cx, cy, s); } else { out_uint8(s, header); - if ((header & FLAGS_NOALPHA) == 0) + red_bytes = cx * cy; + green_bytes = cx * cy; + blue_bytes = cx * cy; + if (header & FLAGS_NOALPHA) + { + out_uint8a(s, red_data, red_bytes); + out_uint8a(s, green_data, green_bytes); + out_uint8a(s, blue_data, blue_bytes); + } + else { + alpha_bytes = cx * cy; out_uint8a(s, alpha_data, alpha_bytes); + out_uint8a(s, red_data, red_bytes); + out_uint8a(s, green_data, green_bytes); + out_uint8a(s, blue_data, blue_bytes); } - out_uint8a(s, red_data, red_bytes); - out_uint8a(s, green_data, green_bytes); - out_uint8a(s, blue_data, blue_bytes); /* pad if no RLE */ out_uint8(s, 0x00); } diff --git a/libxrdp/xrdp_orders.c b/libxrdp/xrdp_orders.c index 7ba6cf01..1412386e 100644 --- a/libxrdp/xrdp_orders.c +++ b/libxrdp/xrdp_orders.c @@ -2327,8 +2327,16 @@ xrdp_orders_send_bitmap(struct xrdp_orders *self, init_stream(temp_s, 16384); p = s->p; i = height; - lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384, - i - 1, temp_s, e); + if (bpp > 24) + { + lines_sending = xrdp_bitmap32_compress(data, width, height, s, bpp, 16384, + i - 1, temp_s, e); + } + else + { + lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384, + i - 1, temp_s, e); + } if (lines_sending != height) {