added rsa_builtin_keygen1 for older openssl libraries

common/ssl_calls.c

@@ -218,6 +218,202 @@ ssl_mod_exp(char* out, int out_len, char* in, int in_len,
   return rv;
 }
 
+#if defined(OLD_RSA_GEN1)
+/*****************************************************************************/
+/* stolen from openssl-0.9.8d, rsa_gen.c, rsa_builtin_keygen
+   RSAerr function calls just commented out
+   returns boolean */
+static int
+rsa_builtin_keygen1(RSA* rsa, int bits, BIGNUM* e_value, BN_GENCB* cb)
+{
+  BIGNUM* r0 = NULL;
+  BIGNUM* r1 = NULL;
+  BIGNUM* r2 = NULL;
+  BIGNUM* r3 = NULL;
+  BIGNUM* tmp;
+  int bitsp;
+  int bitsq;
+  int ok = -1;
+  int n = 0;
+  BN_CTX* ctx = NULL;
+  unsigned int degenerate = 0;
+
+  ctx = BN_CTX_new();
+  if (ctx == NULL)
+  {
+    goto err;
+  }
+  BN_CTX_start(ctx);
+  r0 = BN_CTX_get(ctx);
+  r1 = BN_CTX_get(ctx);
+  r2 = BN_CTX_get(ctx);
+  r3 = BN_CTX_get(ctx);
+  if (r3 == NULL)
+  {
+    goto err;
+  }
+  bitsp = (bits + 1) / 2;
+  bitsq = bits - bitsp;
+  /* We need the RSA components non-NULL */
+  if (!rsa->n && ((rsa->n = BN_new()) == NULL))
+  {
+    goto err;
+  }
+  if (!rsa->d && ((rsa->d = BN_new()) == NULL))
+  {
+    goto err;
+  }
+  if (!rsa->e && ((rsa->e = BN_new()) == NULL))
+  {
+    goto err;
+  }
+  if (!rsa->p && ((rsa->p = BN_new()) == NULL))
+  {
+    goto err;
+  }
+  if (!rsa->q && ((rsa->q = BN_new()) == NULL))
+  {
+    goto err;
+  }
+  if (!rsa->dmp1 && ((rsa->dmp1 = BN_new()) == NULL))
+  {
+    goto err;
+  }
+  if (!rsa->dmq1 && ((rsa->dmq1 = BN_new()) == NULL))
+  {
+    goto err;
+  }
+  if (!rsa->iqmp && ((rsa->iqmp = BN_new()) == NULL))
+  {
+    goto err;
+  }
+  BN_copy(rsa->e, e_value);
+  /* generate p and q */
+  for (;;)
+  {
+    if (!BN_generate_prime_ex(rsa->p, bitsp, 0, NULL, NULL, cb))
+    {
+      goto err;
+    }
+    if (!BN_sub(r2, rsa->p, BN_value_one()))
+    {
+      goto err;
+    }
+    if (!BN_gcd(r1, r2, rsa->e, ctx))
+    {
+      goto err;
+    }
+    if (BN_is_one(r1))
+    {
+      break;
+    }
+    if (!BN_GENCB_call(cb, 2, n++))
+    {
+      goto err;
+    }
+  }
+  if (!BN_GENCB_call(cb, 3, 0))
+  {
+    goto err;
+  }
+  for (;;)
+  {
+    /* When generating ridiculously small keys, we can get stuck
+     * continually regenerating the same prime values. Check for
+     * this and bail if it happens 3 times. */
+    do
+    {
+      if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
+      {
+        goto err;
+      }
+    } while ((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));
+    if (degenerate == 3)
+    {
+      ok = 0; /* we set our own err */
+      /*RSAerr(RSA_F_RSA_BUILTIN_KEYGEN,RSA_R_KEY_SIZE_TOO_SMALL);*/
+      goto err;
+    }
+    if (!BN_sub(r2, rsa->q, BN_value_one()))
+    {
+      goto err;
+    }
+    if (!BN_gcd(r1, r2, rsa->e, ctx))
+    {
+      goto err;
+    }
+    if (BN_is_one(r1))
+    {
+      break;
+    }
+    if (!BN_GENCB_call(cb, 2, n++))
+    {
+      goto err;
+    }
+  }
+  if (!BN_GENCB_call(cb, 3, 1))
+  {
+    goto err;
+  }
+  if (BN_cmp(rsa->p, rsa->q) < 0)
+  {
+    tmp = rsa->p;
+    rsa->p = rsa->q;
+    rsa->q = tmp;
+  }
+  /* calculate n */
+  if (!BN_mul(rsa->n, rsa->p, rsa->q, ctx))
+  {
+    goto err;
+  }
+  /* calculate d */
+  if (!BN_sub(r1, rsa->p, BN_value_one()))
+  {
+    goto err; /* p-1 */
+  }
+  if (!BN_sub(r2, rsa->q, BN_value_one()))
+  {
+    goto err; /* q-1 */
+  }
+  if (!BN_mul(r0, r1, r2, ctx))
+  {
+    goto err; /* (p-1)(q-1) */
+  }
+  if (!BN_mod_inverse(rsa->d, rsa->e, r0, ctx))
+  {
+    goto err; /* d */
+  }
+  /* calculate d mod (p-1) */
+  if (!BN_mod(rsa->dmp1, rsa->d, r1, ctx))
+  {
+    goto err;
+  }
+  /* calculate d mod (q-1) */
+  if (!BN_mod(rsa->dmq1, rsa->d, r2, ctx))
+  {
+    goto err;
+  }
+  /* calculate inverse of q mod p */
+  if (!BN_mod_inverse(rsa->iqmp, rsa->q, rsa->p, ctx))
+  {
+    goto err;
+  }
+  ok = 1;
+err:
+  if (ok == -1)
+  {
+    /*RSAerr(RSA_F_RSA_BUILTIN_KEYGEN,ERR_LIB_BN);*/
+    ok = 0;
+  }
+  if (ctx != NULL)
+  {
+    BN_CTX_end(ctx);
+    BN_CTX_free(ctx);
+  }
+  return ok;
+}
+#endif
+
 /*****************************************************************************/
 /* returns error
    generates a new rsa key
@@ -247,8 +443,7 @@ ssl_gen_key_xrdp1(int key_size_in_bits, char* exp, int exp_len,
   BN_bin2bn((unsigned char*)lexp, exp_len, my_e);
   my_key = RSA_new();
 #if defined(OLD_RSA_GEN1)
-  g_writeln("openssl library old, RSA_generate_key_ex not right");
-  error = 1;
+  error = rsa_builtin_keygen1(my_key, key_size_in_bits, my_e, 0) == 0;
 #else
   error = RSA_generate_key_ex(my_key, key_size_in_bits, my_e, 0) == 0;
 #endif