From b142a59e69ee2b4a721641fe7f5dc9dadb2bb020 Mon Sep 17 00:00:00 2001 From: Jay Sorg Date: Tue, 17 Jun 2014 22:41:10 -0700 Subject: [PATCH] libxrdp: changes for planar compression --- libxrdp/libxrdp.h | 7 +- libxrdp/xrdp_bitmap32_compress.c | 263 ++++++++++++++++++++++++------- libxrdp/xrdp_orders.c | 18 ++- 3 files changed, 216 insertions(+), 72 deletions(-) diff --git a/libxrdp/libxrdp.h b/libxrdp/libxrdp.h index 2ea8b61d..ddfe260b 100644 --- a/libxrdp/libxrdp.h +++ b/libxrdp/libxrdp.h @@ -117,9 +117,10 @@ struct xrdp_sec void *decrypt_rc4_info; void *encrypt_rc4_info; char pub_exp[4]; - char pub_mod[64]; + char pub_mod[256]; char pub_sig[64]; - char pri_exp[64]; + char pri_exp[256]; + int rsa_key_bytes; /* 64 or 256 */ int channel_code; int multimon; char fips_encrypt_key[24]; @@ -509,7 +510,7 @@ int APP_CC xrdp_bitmap32_compress(char *in_data, int width, int height, struct stream *s, int bpp, int byte_limit, int start_line, struct stream *temp_s, - int e); + int e, int flags); int APP_CC xrdp_jpeg_compress(void *handle, char *in_data, int width, int height, struct stream *s, int bpp, int byte_limit, diff --git a/libxrdp/xrdp_bitmap32_compress.c b/libxrdp/xrdp_bitmap32_compress.c index ef20ba0f..d843cff1 100644 --- a/libxrdp/xrdp_bitmap32_compress.c +++ b/libxrdp/xrdp_bitmap32_compress.c @@ -36,9 +36,10 @@ http://msdn.microsoft.com/en-us/library/cc241877.aspx do { if (_level < LLOG_LEVEL) { g_hexdump _args ; } } while (0) /*****************************************************************************/ +/* split ARGB */ static int APP_CC -fsplit(char *in_data, int start_line, int width, int e, - char *alpha_data, char *red_data, char *green_data, char *blue_data) +fsplit4(char *in_data, int start_line, int width, int e, + char *alpha_data, char *red_data, char *green_data, char *blue_data) { #if defined(L_ENDIAN) int alpha; @@ -106,10 +107,10 @@ fsplit(char *in_data, int start_line, int width, int e, } for (index = 0; index < e; index++) { - alpha_data[out_index] = 0; - red_data[out_index] = 0; - green_data[out_index] = 0; - blue_data[out_index] = 0; + alpha_data[out_index] = alpha_data[out_index - 1]; + red_data[out_index] = red_data[out_index - 1]; + green_data[out_index] = green_data[out_index - 1]; + blue_data[out_index] = blue_data[out_index - 1]; out_index++; } start_line--; @@ -119,20 +120,99 @@ fsplit(char *in_data, int start_line, int width, int e, } /*****************************************************************************/ +/* split RGB */ static int APP_CC -fdelta(char *plane, int cx, int cy) +fsplit3(char *in_data, int start_line, int width, int e, + char *red_data, char *green_data, char *blue_data) +{ +#if defined(L_ENDIAN) + int red; + int green; + int blue; +#endif + int index; + int out_index; + int pixel; + int cy; + int *ptr32; + + cy = 0; + out_index = 0; + while (start_line >= 0) + { + ptr32 = (int *) (in_data + start_line * width * 4); + index = 0; +#if defined(L_ENDIAN) + while (index + 4 <= width) + { + pixel = *ptr32; + ptr32++; + red = (pixel >> 16) & 0x000000ff; + green = (pixel >> 8) & 0x000000ff; + blue = (pixel >> 0) & 0x000000ff; + pixel = *ptr32; + ptr32++; + red |= (pixel >> 8) & 0x0000ff00; + green |= (pixel << 0) & 0x0000ff00; + blue |= (pixel << 8) & 0x0000ff00; + pixel = *ptr32; + ptr32++; + red |= (pixel >> 0) & 0x00ff0000; + green |= (pixel << 8) & 0x00ff0000; + blue |= (pixel << 16) & 0x00ff0000; + pixel = *ptr32; + ptr32++; + red |= (pixel << 8) & 0xff000000; + green |= (pixel << 16) & 0xff000000; + blue |= (pixel << 24) & 0xff000000; + *((int*)(red_data + out_index)) = red; + *((int*)(green_data + out_index)) = green; + *((int*)(blue_data + out_index)) = blue; + out_index += 4; + index += 4; + } +#endif + while (index < width) + { + pixel = *ptr32; + ptr32++; + red_data[out_index] = pixel >> 16; + green_data[out_index] = pixel >> 8; + blue_data[out_index] = pixel >> 0; + out_index++; + index++; + } + for (index = 0; index < e; index++) + { + red_data[out_index] = red_data[out_index - 1]; + green_data[out_index] = green_data[out_index - 1]; + blue_data[out_index] = blue_data[out_index - 1]; + out_index++; + } + start_line--; + cy++; + } + return cy; +} + +/*****************************************************************************/ +static int APP_CC +fdelta(char *in_plane, char *out_plane, int cx, int cy) { char delta; - char *ptr8; + char *src8; + char *dst8; int index; int jndex; + g_memcpy(out_plane, in_plane, cx); for (jndex = cy - 2; jndex >= 0; jndex--) { - ptr8 = plane + jndex * cx; + src8 = in_plane + jndex * cx; + dst8 = out_plane + jndex * cx; for (index = 0; index < cx; index++) { - delta = ptr8[cx] - ptr8[0]; + delta = src8[cx] - src8[0]; if (delta & 0x80) { delta = (((~delta) + 1) << 1) - 1; @@ -141,8 +221,9 @@ fdelta(char *plane, int cx, int cy) { delta = delta << 1; } - ptr8[cx] = delta; - ptr8++; + dst8[cx] = delta; + src8++; + dst8++; } } return 0; @@ -278,99 +359,159 @@ fpack(char *plane, int cx, int cy, struct stream *s) return (int) (s->p - holdp); } +/*****************************************************************************/ +static int APP_CC +foutraw3(struct stream *s, int bytes, int header, + char *red_data, char *green_data, char *blue_data) +{ + out_uint8(s, header); + out_uint8a(s, red_data, bytes); + out_uint8a(s, green_data, bytes); + out_uint8a(s, blue_data, bytes); + /* pad if no RLE */ + out_uint8(s, 0x00); + return 0; +} + +/*****************************************************************************/ +static int APP_CC +foutraw4(struct stream *s, int bytes, int header, + char *alpha_data, char *red_data, char *green_data, char *blue_data) +{ + out_uint8(s, header); + out_uint8a(s, alpha_data, bytes); + out_uint8a(s, red_data, bytes); + out_uint8a(s, green_data, bytes); + out_uint8a(s, blue_data, bytes); + /* pad if no RLE */ + out_uint8(s, 0x00); + return 0; +} + /*****************************************************************************/ /* returns the number of lines compressed */ int APP_CC xrdp_bitmap32_compress(char *in_data, int width, int height, struct stream *s, int bpp, int byte_limit, int start_line, struct stream *temp_s, - int e) + int e, int flags) { char *alpha_data; char *red_data; char *green_data; char *blue_data; + char *salpha_data; + char *sred_data; + char *sgreen_data; + char *sblue_data; int alpha_bytes; int red_bytes; int green_bytes; int blue_bytes; int cx; int cy; - int header; int max_bytes; + int total_bytes; + int header; LLOGLN(10, ("xrdp_bitmap32_compress:")); - //header = FLAGS_NOALPHA | FLAGS_RLE; - //header = FLAGS_NOALPHA; - header = FLAGS_RLE; - + header = flags & 0xFF; cx = width + e; - alpha_data = temp_s->data; + salpha_data = temp_s->data; + sred_data = salpha_data + cx * height; + sgreen_data = sred_data + cx * height; + sblue_data = sgreen_data + cx * height; + alpha_data = sblue_data + cx * height; red_data = alpha_data + cx * height; green_data = red_data + cx * height; blue_data = green_data + cx * height; - /* split planes */ - cy = fsplit(in_data, start_line, width, e, - alpha_data, red_data, green_data, blue_data); - - if (header & FLAGS_RLE) + if (header & FLAGS_NOALPHA) { - out_uint8(s, header); - if (header & FLAGS_NOALPHA) + cy = fsplit3(in_data, start_line, width, e, + sred_data, sgreen_data, sblue_data); + if (header & FLAGS_RLE) { - fdelta(red_data, cx, cy); - fdelta(green_data, cx, cy); - fdelta(blue_data, cx, cy); - alpha_bytes = 0; + fdelta(sred_data, red_data, cx, cy); + fdelta(sgreen_data, green_data, cx, cy); + fdelta(sblue_data, blue_data, cx, cy); + out_uint8(s, header); red_bytes = fpack(red_data, cx, cy, s); green_bytes = fpack(green_data, cx, cy, s); blue_bytes = fpack(blue_data, cx, cy, s); + total_bytes = red_bytes + green_bytes + blue_bytes; + if (1 + total_bytes > byte_limit) + { + /* failed */ + LLOGLN(0, ("xrdp_bitmap32_compress: too big, rgb " + "bytes %d %d %d total_bytes %d cx %d cy %d byte_limit %d", + red_bytes, green_bytes, blue_bytes, + total_bytes, cx, cy, byte_limit)); + return 0; + } max_bytes = cx * cy * 3; + if (total_bytes > max_bytes) + { + /* raw is better */ + LLOGLN(10, ("xrdp_bitmap32_compress: too big, rgb " + "bytes %d %d %d total_bytes %d cx %d cy %d max_bytes %d", + red_bytes, green_bytes, blue_bytes, + total_bytes, cx, cy, max_bytes)); + init_stream(s, 0); + foutraw3(s, cx * cy, FLAGS_NOALPHA, sred_data, + sgreen_data, sblue_data); + } } else { - fdelta(alpha_data, cx, cy); - fdelta(red_data, cx, cy); - fdelta(green_data, cx, cy); - fdelta(blue_data, cx, cy); - alpha_bytes = fpack(alpha_data, cx, cy, s); - red_bytes = fpack(red_data, cx, cy, s); - green_bytes = fpack(green_data, cx, cy, s); - blue_bytes = fpack(blue_data, cx, cy, s); - max_bytes = cx * cy * 4; - } - if (alpha_bytes + red_bytes + green_bytes + blue_bytes > max_bytes) - { - LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb " - "bytes %d %d %d %d cx %d cy %d", alpha_bytes, red_bytes, - green_bytes, blue_bytes, cx, cy)); + foutraw3(s, cx * cy, FLAGS_NOALPHA, sred_data, + sgreen_data, sblue_data); } } else { - out_uint8(s, header); - red_bytes = cx * cy; - green_bytes = cx * cy; - blue_bytes = cx * cy; - if (header & FLAGS_NOALPHA) + cy = fsplit4(in_data, start_line, width, e, + salpha_data, sred_data, sgreen_data, sblue_data); + if (header & FLAGS_RLE) { - out_uint8a(s, red_data, red_bytes); - out_uint8a(s, green_data, green_bytes); - out_uint8a(s, blue_data, blue_bytes); + fdelta(salpha_data, alpha_data, cx, cy); + fdelta(sred_data, red_data, cx, cy); + fdelta(sgreen_data, green_data, cx, cy); + fdelta(sblue_data, blue_data, cx, cy); + out_uint8(s, header); + alpha_bytes = fpack(alpha_data, cx, cy, s); + red_bytes = fpack(red_data, cx, cy, s); + green_bytes = fpack(green_data, cx, cy, s); + blue_bytes = fpack(blue_data, cx, cy, s); + max_bytes = cx * cy * 4; + total_bytes = alpha_bytes + red_bytes + green_bytes + blue_bytes; + if (1 + total_bytes > byte_limit) + { + /* failed */ + LLOGLN(0, ("xrdp_bitmap32_compress: too big, argb " + "bytes %d %d %d %d total_bytes %d cx %d cy %d byte_limit %d", + alpha_bytes, red_bytes, green_bytes, blue_bytes, + total_bytes, cx, cy, byte_limit)); + return 0; + } + if (total_bytes > max_bytes) + { + /* raw is better */ + LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb " + "bytes %d %d %d %d total_bytes %d cx %d cy %d max_bytes %d", + alpha_bytes, red_bytes, green_bytes, blue_bytes, + total_bytes, cx, cy, max_bytes)); + init_stream(s, 0); + foutraw4(s, cx * cy, 0, salpha_data, sred_data, + sgreen_data, sblue_data); + } } else { - alpha_bytes = cx * cy; - out_uint8a(s, alpha_data, alpha_bytes); - out_uint8a(s, red_data, red_bytes); - out_uint8a(s, green_data, green_bytes); - out_uint8a(s, blue_data, blue_bytes); + foutraw4(s, cx * cy, 0, salpha_data, sred_data, + sgreen_data, sblue_data); } - /* pad if no RLE */ - out_uint8(s, 0x00); } - return cy; } diff --git a/libxrdp/xrdp_orders.c b/libxrdp/xrdp_orders.c index 1412386e..88ffda8e 100644 --- a/libxrdp/xrdp_orders.c +++ b/libxrdp/xrdp_orders.c @@ -2322,15 +2322,16 @@ xrdp_orders_send_bitmap(struct xrdp_orders *self, } make_stream(s); - init_stream(s, 16384); + init_stream(s, 16384 * 2); make_stream(temp_s); - init_stream(temp_s, 16384); + init_stream(temp_s, 16384 * 2); p = s->p; i = height; if (bpp > 24) { - lines_sending = xrdp_bitmap32_compress(data, width, height, s, bpp, 16384, - i - 1, temp_s, e); + lines_sending = xrdp_bitmap32_compress(data, width, height, s, + bpp, 16384, + i - 1, temp_s, e, 0x30); } else { @@ -2590,15 +2591,16 @@ xrdp_orders_send_bitmap2(struct xrdp_orders *self, } make_stream(s); - init_stream(s, 16384); + init_stream(s, 16384 * 2); make_stream(temp_s); - init_stream(temp_s, 16384); + init_stream(temp_s, 16384 * 2); p = s->p; i = height; if (bpp > 24) { - lines_sending = xrdp_bitmap32_compress(data, width, height, s, bpp, 16384, - i - 1, temp_s, e); + lines_sending = xrdp_bitmap32_compress(data, width, height, s, + bpp, 16384, + i - 1, temp_s, e, 0x30); } else {