added support for if OLD_RSA_GEN1 is defined and changed unsigned char to tui8

common/ssl_calls.c

@@ -57,14 +57,14 @@ ssl_rc4_info_delete(void* rc4_info)
 void APP_CC
 ssl_rc4_set_key(void* rc4_info, char* key, int len)
 {
-  RC4_set_key((RC4_KEY*)rc4_info, len, (unsigned char*)key);
+  RC4_set_key((RC4_KEY*)rc4_info, len, (tui8*)key);
 }
 
 /*****************************************************************************/
 void APP_CC
 ssl_rc4_crypt(void* rc4_info, char* data, int len)
 {
-  RC4((RC4_KEY*)rc4_info, len, (unsigned char*)data, (unsigned char*)data);
+  RC4((RC4_KEY*)rc4_info, len, (tui8*)data, (tui8*)data);
 }
 
 /* sha1 stuff */
@@ -101,7 +101,7 @@ ssl_sha1_transform(void* sha1_info, char* data, int len)
 void APP_CC
 ssl_sha1_complete(void* sha1_info, char* data)
 {
-  SHA1_Final((unsigned char*)data, (SHA_CTX*)sha1_info);
+  SHA1_Final((tui8*)data, (SHA_CTX*)sha1_info);
 }
 
 /* md5 stuff */
@@ -138,7 +138,7 @@ ssl_md5_transform(void* md5_info, char* data, int len)
 void APP_CC
 ssl_md5_complete(void* md5_info, char* data)
 {
-  MD5_Final((unsigned char*)data, (MD5_CTX*)md5_info);
+  MD5_Final((tui8*)data, (MD5_CTX*)md5_info);
 }
 
 /*****************************************************************************/
@@ -192,11 +192,11 @@ ssl_mod_exp(char* out, int out_len, char* in, int in_len,
   BN_init(&lexp);
   BN_init(&lin);
   BN_init(&lout);
-  BN_bin2bn((unsigned char*)l_mod, mod_len, &lmod);
-  BN_bin2bn((unsigned char*)l_exp, exp_len, &lexp);
-  BN_bin2bn((unsigned char*)l_in, in_len, &lin);
+  BN_bin2bn((tui8*)l_mod, mod_len, &lmod);
+  BN_bin2bn((tui8*)l_exp, exp_len, &lexp);
+  BN_bin2bn((tui8*)l_in, in_len, &lin);
   BN_mod_exp(&lout, &lin, &lexp, &lmod, ctx);
-  rv = BN_bn2bin(&lout, (unsigned char*)l_out);
+  rv = BN_bn2bin(&lout, (tui8*)l_out);
   if (rv <= out_len)
   {
     ssl_reverse_it(l_out, rv);
@@ -220,12 +220,65 @@ ssl_mod_exp(char* out, int out_len, char* in, int in_len,
 
 #if defined(OLD_RSA_GEN1)
 /*****************************************************************************/
-/* returns error, always errors for now */
+/* returns error
+   generates a new rsa key
+   exp is passed in and mod and pri are passed out */
 int APP_CC
 ssl_gen_key_xrdp1(int key_size_in_bits, char* exp, int exp_len,
                   char* mod, int mod_len, char* pri, int pri_len)
 {
-  return 1;
+  int my_e;
+  RSA* my_key;
+  char* lmod;
+  char* lpri;
+  tui8* lexp;
+  int error;
+  int len;
+
+  if ((exp_len != 4) || (mod_len != 64) || (pri_len != 64))
+  {
+    return 1;
+  }
+  lmod = (char*)g_malloc(mod_len, 0);
+  lpri = (char*)g_malloc(pri_len, 0);
+  lexp = (tui8*)exp;
+  my_e = lexp[0];
+  my_e |= lexp[1] << 8;
+  my_e |= lexp[2] << 16;
+  my_e |= lexp[3] << 24;
+  /* srand is in stdlib.h*/
+  srand(g_time1());
+  my_key = RSA_generate_key(key_size_in_bits, my_e, 0, 0);
+  error = my_key == 0;
+  if (error == 0)
+  {
+    len = BN_num_bytes(my_key->n);
+    error = len != mod_len;
+  }
+  if (error == 0)
+  {
+    BN_bn2bin(my_key->n, (tui8*)lmod);
+    ssl_reverse_it(lmod, mod_len);
+  }
+  if (error == 0)
+  {
+    len = BN_num_bytes(my_key->d);
+    error = len != pri_len;
+  }
+  if (error == 0)
+  {
+    BN_bn2bin(my_key->d, (tui8*)lpri);
+    ssl_reverse_it(lpri, pri_len);
+  }
+  if (error == 0)
+  {
+    g_memcpy(mod, lmod, mod_len);
+    g_memcpy(pri, lpri, pri_len);
+  }
+  RSA_free(my_key);
+  g_free(lmod);
+  g_free(lpri);
+  return error;
 }
 #else
 /*****************************************************************************/
@@ -254,7 +307,7 @@ ssl_gen_key_xrdp1(int key_size_in_bits, char* exp, int exp_len,
   g_memcpy(lexp, exp, exp_len);
   ssl_reverse_it(lexp, exp_len);
   my_e = BN_new();
-  BN_bin2bn((unsigned char*)lexp, exp_len, my_e);
+  BN_bin2bn((tui8*)lexp, exp_len, my_e);
   my_key = RSA_new();
   error = RSA_generate_key_ex(my_key, key_size_in_bits, my_e, 0) == 0;
   if (error == 0)
@@ -264,7 +317,7 @@ ssl_gen_key_xrdp1(int key_size_in_bits, char* exp, int exp_len,
   }
   if (error == 0)
   {
-    BN_bn2bin(my_key->n, (unsigned char*)lmod);
+    BN_bn2bin(my_key->n, (tui8*)lmod);
     ssl_reverse_it(lmod, mod_len);
   }
   if (error == 0)
@@ -274,7 +327,7 @@ ssl_gen_key_xrdp1(int key_size_in_bits, char* exp, int exp_len,
   }
   if (error == 0)
   {
-    BN_bn2bin(my_key->d, (unsigned char*)lpri);
+    BN_bn2bin(my_key->d, (tui8*)lpri);
     ssl_reverse_it(lpri, pri_len);
   }
   if (error == 0)