libsec: refactor asn1 encoding of digest for rsa signatures, fix memory leak in ecverify

sys/src/libsec/port/ecc.c

@@ -476,6 +476,7 @@ ecdsaverify(ECdomain *dom, ECpub *pub, uchar *dig, int len, mpint *r, mpint *s)
 		mpmod(R.x, dom->n, t);
 		ret = mpcmp(r, t) == 0;
 	}
+	mpfree(E);
 	mpfree(t);
 	mpfree(u1);
 	mpfree(u2);

sys/src/libsec/port/tlshand.c

@@ -432,7 +432,7 @@ static void freeints(Ints* b);
 
 /* x509.c */
 extern mpint*	pkcs1padbuf(uchar *buf, int len, mpint *modulus);
-extern int	X509encodesignature_sha256(uchar digest[SHA2_256dlen], uchar *buf, int len);
+extern int	asn1encodedigest(DigestState* (*fun)(uchar*, ulong, uchar*, DigestState*), uchar *digest, uchar *buf, int len);
 
 //================= client/server ========================
 
@@ -1298,13 +1298,18 @@ tlsClient2(int ctl, int hand,
 
 			m.u.certificateVerify.sigalg = 0x0401;	/* RSA SHA256 */
 			sha2_256(nil, 0, digest, &c->handhash.sha2_256);
-			buflen = X509encodesignature_sha256(digest, buf, sizeof(buf));
+			buflen = asn1encodedigest(sha2_256, digest, buf, sizeof(buf));
 		} else {
 			md5(nil, 0, buf, &c->handhash.md5);
 			sha1(nil, 0, buf+MD5dlen, &c->handhash.sha1);
 			buflen = MD5dlen+SHA1dlen;
 		}
 		c->handhash = hsave;
+
+		if(buflen <= 0){
+			tlsError(c, EInternalError, "can't encode handshake hashes");
+			goto Err;
+		}
 		
 		paddedHashes = pkcs1padbuf(buf, buflen, c->sec->rsapub->n);
 		signedMP = factotum_rsa_decrypt(c->sec->rpc, paddedHashes);

sys/src/libsec/port/x509.c

@@ -1896,7 +1896,7 @@ errret:
 }
 
 /*
- *	RSAPublickKey :: SEQUENCE {
+ *	RSAPublickKey ::= SEQUENCE {
  *		modulus INTEGER,
  *		publicExponent INTEGER
  *	}
@@ -2552,30 +2552,52 @@ mkDN(char *dn)
 	return mkseq(el);
 }
 
-int
-X509encodesignature_sha256(uchar digest[SHA2_256dlen], uchar *buf, int len)
+/*
+ * DigestInfo ::= SEQUENCE {
+ *	digestAlgorithm AlgorithmIdentifier,
+ *	digest OCTET STRING }
+ */
+static Bytes*
+encode_digest(DigestAlg *da, uchar *digest)
 {
-	Bytes *sigbytes;
-	Elem sig;
+	Bytes *ans;
 	int err;
+	Elem e;
 
-	sig = mkseq(
-		mkel(mkalg(ALG_sha256),
-		mkel(mkoctet(digest, SHA2_256dlen),
+	e = mkseq(
+		mkel(mkalg(da->alg),
+		mkel(mkoctet(digest, da->len),
 		nil)));
-	err = encode(sig, &sigbytes);
-	freevalfields(&sig.val);
+	err = encode(e, &ans);
+	freevalfields(&e.val);
 	if(err != ASN_OK)
-		return -1;
-	if(len < sigbytes->len){
-		freebytes(sigbytes);
-		return -1;
-	}
-	len = sigbytes->len;
-	memmove(buf, sigbytes->data, len);
-	freebytes(sigbytes);
+		return nil;
 
-	return len;
+	return ans;
+}
+
+int
+asn1encodedigest(DigestState* (*fun)(uchar*, ulong, uchar*, DigestState*), uchar *digest, uchar *buf, int len)
+{
+	Bytes *bytes;
+	DigestAlg **dp;
+
+	for(dp = digestalg; *dp != nil; dp++){
+		if((*dp)->fun != fun)
+			continue;
+		bytes = encode_digest(*dp, digest);
+		if(bytes == nil)
+			break;
+		if(bytes->len > len){
+			freebytes(bytes);
+			break;
+		}
+		len = bytes->len;
+		memmove(buf, bytes->data, len);
+		freebytes(bytes);
+		return len;
+	}
+	return -1;
 }
 
 uchar*
@@ -2612,21 +2634,18 @@ X509rsagen(RSApriv *priv, char *subj, ulong valid[2], int *certlen)
 	freebytes(pkbytes);
 	if(encode(e, &certinfobytes) != ASN_OK)
 		goto errret;
+
 	da = digestalg[sigalg];
 	(*da->fun)(certinfobytes->data, certinfobytes->len, digest, 0);
 	freebytes(certinfobytes);
 	certinfo = e;
-	e = mkseq(
-		mkel(mkalg(da->alg),
-		mkel(mkoctet(digest, da->len),
-		nil)));
-	if(encode(e, &sigbytes) != ASN_OK){
-		freevalfields(&certinfo.val);
+
+	sigbytes = encode_digest(da, digest);
+	if(sigbytes == nil)
 		goto errret;
-	}
-	freevalfields(&e.val);
 	pkcs1 = pkcs1pad(sigbytes, pk->n);
 	freebytes(sigbytes);
+
 	rsadecrypt(priv, pkcs1, pkcs1);
 	buflen = mptobe(pkcs1, nil, 0, &buf);
 	mpfree(pkcs1);
@@ -2682,16 +2701,13 @@ X509rsareq(RSApriv *priv, char *subj, int *certlen)
 	(*da->fun)(certinfobytes->data, certinfobytes->len, digest, 0);
 	freebytes(certinfobytes);
 	certinfo = e;
-	e = mkseq(
-		mkel(mkalg(da->alg),
-		mkel(mkoctet(digest, da->len),
-		nil)));
-	if(encode(e, &sigbytes) != ASN_OK){
-		freevalfields(&certinfo.val);
+
+	sigbytes = encode_digest(da, digest);
+	if(sigbytes == nil)
 		goto errret;
-	}
 	pkcs1 = pkcs1pad(sigbytes, pk->n);
 	freebytes(sigbytes);
+
 	rsadecrypt(priv, pkcs1, pkcs1);
 	buflen = mptobe(pkcs1, nil, 0, &buf);
 	mpfree(pkcs1);