RSA key generation in two steps.

ChangeLog

@@ -1,3 +1,11 @@
+2017-10-04  NIIBE Yutaka  <gniibe@fsij.org>
+
+	* src/openpgp-do.c (gpg_do_keygen): Do RSA key generation in two
+	steps.
+
+	* src/call-rsa.c (rsa_genkey_start, rsa_genkey_finish): New.
+	(rsa_genkey): Remove.
+
 2017-10-03  NIIBE Yutaka  <gniibe@fsij.org>
 
 	* src/call-ec.c (ecc_compute_public): No use of malloc.

src/call-rsa.c

@@ -237,12 +237,10 @@ rsa_verify (const uint8_t *pubkey, int pubkey_len,
 #define RSA_EXPONENT 0x10001
 
 int
-rsa_genkey (int pubkey_len, uint8_t *pubkey, uint8_t *p_q)
+rsa_genkey_start (int pubkey_len)
 {
   int ret;
   uint8_t index = 0;
-  uint8_t *p = p_q;
-  uint8_t *q = p_q + pubkey_len / 2;
   int cs;
 
   extern int prng_seed (int (*f_rng)(void *, unsigned char *, size_t),
@@ -260,14 +258,30 @@ rsa_genkey (int pubkey_len, uint8_t *pubkey, uint8_t *p_q)
   cs = chopstx_setcancelstate (0); /* Allow cancellation.  */
   MPI_CHK( rsa_gen_key (&rsa_ctx, random_gen, &index, pubkey_len * 8,
 			RSA_EXPONENT) );
-  MPI_CHK( mpi_write_binary (&rsa_ctx.P, p, pubkey_len / 2) );
-  MPI_CHK( mpi_write_binary (&rsa_ctx.Q, q, pubkey_len / 2) );
-  MPI_CHK( mpi_write_binary (&rsa_ctx.N, pubkey, pubkey_len) );
-  clp.arg = NULL;
 
  cleanup:
   chopstx_setcancelstate (cs);
-  chopstx_cleanup_pop (1);
+  chopstx_cleanup_pop (0);
+  if (ret != 0)
+    return -1;
+  else
+    return 0;
+}
+
+int
+rsa_genkey_finish (int pubkey_len, uint8_t *pubkey, uint8_t *p_q)
+{
+  int ret;
+  uint8_t *p = p_q;
+  uint8_t *q = p_q + pubkey_len / 2;
+
+  MPI_CHK( mpi_write_binary (&rsa_ctx.P, p, pubkey_len / 2) );
+  MPI_CHK( mpi_write_binary (&rsa_ctx.Q, q, pubkey_len / 2) );
+  MPI_CHK( mpi_write_binary (&rsa_ctx.N, pubkey, pubkey_len) );
+
+ cleanup:
+  rsa_free (&rsa_ctx);
+
   if (ret != 0)
     return -1;
   else

src/gnuk.h

@@ -269,7 +269,8 @@ int modulus_calc (const uint8_t *, int, uint8_t *);
 int rsa_decrypt (const uint8_t *, uint8_t *, int, struct key_data *,
 		 unsigned int *);
 int rsa_verify (const uint8_t *, int, const uint8_t *, const uint8_t *);
-int rsa_genkey (int, uint8_t *, uint8_t *);
+int rsa_genkey_start (int);
+int rsa_genkey_finish (int, uint8_t *, uint8_t *);
 
 int ecdsa_sign_p256r1 (const uint8_t *hash, uint8_t *output,
 		       const uint8_t *key_data);

src/openpgp-do.c

@@ -2063,25 +2063,31 @@ gpg_do_keygen (uint8_t kk_byte)
   enum kind_of_key kk = kkb_to_kk (kk_byte);
   int attr = gpg_get_algo_attr (kk);;
   int prvkey_len = gpg_get_algo_attr_key_size (kk, GPG_KEY_PRIVATE);
-  const uint8_t *keystring_admin;
-  const uint8_t *rnd;
-  const uint8_t *prv;
-  uint8_t d[64];
-  uint8_t p_q[512];
-  uint8_t pubkey[512];
   int r = 0;
 
   DEBUG_INFO ("Keygen\r\n");
   DEBUG_BYTE (kk_byte);
 
-  if (admin_authorized == BY_ADMIN)
+  /* RSA key generation is done in two steps to lower memory pressure.  */
-    keystring_admin = keystring_md_pw3;
+  if (attr == ALGO_RSA2K || attr == ALGO_RSA4K)
-  else
+    {
-    keystring_admin = NULL;
+      if (rsa_genkey_start (prvkey_len) < 0)
+	{
+	  GPG_MEMORY_FAILURE ();
+	  return;
+	}
+    }
+
+  {
+    const uint8_t *prv;
+    const uint8_t *rnd;
+    uint8_t d[64];
+    uint8_t p_q[512];
+    uint8_t pubkey[512];
 
     if (attr == ALGO_RSA2K || attr == ALGO_RSA4K)
       {
-      if (rsa_genkey (prvkey_len, pubkey, p_q) < 0)
+	if (rsa_genkey_finish (prvkey_len, pubkey, p_q) < 0)
 	  {
 	    GPG_MEMORY_FAILURE ();
 	    return;
@@ -2154,8 +2160,20 @@ gpg_do_keygen (uint8_t kk_byte)
       }
 
     if (r >= 0)
+      {
+	const uint8_t *keystring_admin;
+
+	if (admin_authorized == BY_ADMIN)
+	  keystring_admin = keystring_md_pw3;
+	else
+	  keystring_admin = NULL;
+
 	r = gpg_do_write_prvkey (kk, prv, prvkey_len, keystring_admin, pubkey);
+      }
+
     /* XXX: clear private key data on stack here.  */
+  }
+
   if (r < 0)
     {
       GPG_ERROR ();